Description #

Excel spreadsheet parsing processor that supports multi-format output and structured data extraction

Key features #

• Supports parsing of xlsx/xls formats
• Outputs in markdown or HTML format
• Preserves table structure and data relationships
• Supports processing of multiple worksheets
• Provides TOS storage options

Format support #

• Microsoft Excel (.xlsx, .xls)
• It is recommended to use the xlsx format for optimal parsing results

Operator parameters #

Input #

Output #

Array of structs, with the following fields

• data_item_uri: Original file path
• text: Merged markdown/HTML text
• text_by_table: List of markdown/HTML text for each sheet

Parameters #

If a parameter does not have a default value, it is required

Examples #

The following code demonstrates how to use pandas (on a single machine) and ray (in a distributed environment) to run the operator to parse xlsx documents.

from __future__ import annotations

import os

import daft
from daft import col
from daft.las.functions.doc import XlsxParse
from daft.las.functions.udf import las_udf

if __name__ == "__main__":

    # After making changes, the parsed content will be saved to the specified TOS path. Therefore, you need to set the environment variables to ensure write permissions to TOS, including: ACCESS_KEY, SECRET_KEY, TOS_ENDPOINT, TOS_REGION, TOS_TEST_DIR
    TOS_DIR = os.getenv("TOS_TEST_DIR", "tos_bucket")
    output_tos_path = f"tos://{TOS_DIR}/doc/parse/xlsx_parse"
    
    if os.getenv("DAFT_RUNNER", "native") == "ray":
        import logging

        import ray

        def configure_logging():
            logging.basicConfig(
                level=logging.INFO,
                format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
                datefmt="%Y-%m-%d %H:%M:%S.%s".format(),
            )
            logging.getLogger("tracing.span").setLevel(logging.WARNING)
            logging.getLogger("daft_io.stats").setLevel(logging.WARNING)
            logging.getLogger("DaftStatisticsManager").setLevel(logging.WARNING)
            logging.getLogger("DaftFlotillaScheduler").setLevel(logging.WARNING)
            logging.getLogger("DaftFlotillaDispatcher").setLevel(logging.WARNING)

        ray.init(dashboard_host="0.0.0.0", runtime_env={"worker_process_setup_hook": configure_logging})
        daft.set_runner_ray()
    daft.set_execution_config(actor_udf_ready_timeout=600)
    daft.set_execution_config(min_cpu_per_task=0)

    # Use absolute URLs to eliminate the dependency on environment variables.
    tos_dir_url = os.getenv("TOS_DIR_URL", "las-cn-beijing-public-online.tos-cn-beijing.volces.com")
    samples = {
        "xlsx_path": [
            f"https://{tos_dir_url}/public/shared_doc_dataset/sample.xlsx"
        ],
    }
    df = daft.from_pydict(samples)

    constructor_kwargs = {
        "if_save_md_content": True,
        "if_save_html_content": False,
        "output_tos_path": output_tos_path,
    }

    # Use Daft for distributed processing
    df = df.with_column(
        "result",
        las_udf(XlsxParse, construct_args=constructor_kwargs, concurrency=1)(col("xlsx_path")),
    )
    df = df.with_column("data_item_uri", col("result")["data_item_uri"])
    df = df.with_column("text", col("result")["text"])
    df = df.with_column("text_by_table", col("result")["text_by_table"])

    df.show()
    # ╭────────────────────────────────┬────────────────────────────────────────────────────────────────────┬────────────────────────────────┬────────────────────────────────────────┬────────────────────────────────────────╮
    # │ xlsx_path                      ┆ result                                                             ┆ data_item_uri                  ┆ text                                   ┆ text_by_table                          │
    # │ ---                            ┆ ---                                                                ┆ ---                            ┆ ---                                    ┆ ---                                    │
    # │ Utf8                           ┆ Struct[data_item_uri: Utf8, text: Utf8, text_by_table: List[Utf8]] ┆ Utf8                           ┆ Utf8                                   ┆ List[Utf8]                             │
    # ╞════════════════════════════════╪════════════════════════════════════════════════════════════════════╪════════════════════════════════╪════════════════════════════════════════╪════════════════════════════════════════╡
    # │ https://las-cn-beijing-publi-… ┆ {data_item_uri: tos://tos_bucket/doc/parse/xlsx_parse/sample.md, … ┆ tos://tos_bucket/doc/parse/xls ┆ |   产品ID | 产品名称   |   价格 |   …    ┆ [|   产品ID | 产品名称   |   价格 |  …    │
    # ╰────────────────────────────────┴────────────────────────────────────────────────────────────────────┴────────────────────────────────┴────────────────────────────────────────┴────────────────────────────────────────╯