polars-bio-fast, scalable, and out-of-core operations on large genomic interval datasets.

{"sci_version": "1.0", "schema": "https://corpus.bntrg.co.uk/sci/1.0/sci.schema.json", "id": "pmid:41325273", "kind": "science-record", "revision": 1, "content_hash": "sha256:f842281bfcc6937693b2f9783835356a0d22bffac3e337ad0dc8949d4e0a7453", "created_at": "2026-11-01T20:31:20Z", "updated_at": "2026-11-01T20:31:20Z", "generator": {"name": "corpus-assay-engine", "run_id": "reading-sci-2026-11-01"}, "paper": {"state": "complete", "title": "polars-bio-fast, scalable, and out-of-core operations on large genomic interval datasets.", "doi": "10.1093/bioinformatics/btaf640", "pmid": "41325273", "authors": ["Marek Wiewiórka", "Pavel Khamutou", "Marek Zbysiński", "Tomasz Gambin"], "journal": "Bioinformatics (Oxford, England)", "year": 2025, "abstract": "MOTIVATION: Genomic studies very often rely on computationally intensive analyses of relationships between features, which are typically represented as intervals along a 1D coordinate system (such as positions on a chromosome). In this context, the Python programming language is extensively used for manipulating and analyzing data stored in a tabular form of rows and columns, called a DataFrame. Pandas is the most widely used Python DataFrame package and has been criticized for inefficiencies and scalability issues, which its modern alternative-Polars-aims to address with a native backend written in the Rust programming language.\nRESULTS: polars-bio is a Python library that enables fast, parallel and out-of-core operations on large genomic interval datasets. Its main components are implemented in Rust, using the Apache DataFusion query engine and Apache Arrow for efficient data representation. It is compatible with Polars and Pandas DataFrame formats. In a real-world comparison (107 versus 1.2×106 intervals), our library runs overlap queries 6.5×, nearest queries 15.5×, count_overlaps queries 38×, and coverage queries 15× faster than Bioframe. On equally sized synthetic sets (107 versus 107), the corresponding speedups are 1.6×, 5.5×, 6×, and 6×. In streaming mode, on real and synthetic interval pairs, our implementation uses 90× and 15× less memory for overlap, 4.5× and 6.5× less for nearest, 60× and 12× less for count_overlaps, and 34× and 7× less for coverage than Bioframe. Multi-threaded benchmarks show good scalability characteristics. To the best of our knowledge, polars-bio is the most efficient single-node library for genomic interval DataFrames in Python.\nAVAILABILITY AND IMPLEMENTATION: polars-bio is an open-source Python package distributed under the Apache License available for major platforms, including Linux, macOS, and Windows in the PyPI registry. The online documentation is https://biodatageeks.org/polars-bio/ and the source code is available on GitHub: https://github.com/biodatageeks/polars-bio and Zenodo: https://doi.org/10.5281/zenodo.16374290. are available at Bioinformatics online.", "claims": [{"id": "claim-data-access", "statement": "Links to all datasets used in this study are provided in Section 3.2 of the Supplementary Materials.", "kind": "methodological", "target": null, "evidence": [{"asset": "source:pmid:41325273", "locator": "sections[14] \"Data availability\" paragraph 2", "quote": "Links to all datasets used in this study are provided in Section 3.2 of the Supplementary Materials."}], "state": "complete"}, {"id": "claim-target-result", "statement": "In a real-world comparison (107 versus 1.2×106 intervals), our library runs overlap queries 6.5×, nearest queries 15.5×, count_overlaps queries 38×, and coverage queries 15× faster than Bioframe.", "kind": "qualitative", "target": null, "evidence": [{"asset": "source:pmid:41325273", "locator": "sections[0] \"Abstract\" paragraph 2", "quote": "In a real-world comparison (107 versus 1.2×106 intervals), our library runs overlap queries 6.5×, nearest queries 15.5×, count_overlaps queries 38×, and coverage queries 15× faster than Bioframe."}], "state": "complete"}], "gaps": []}, "experiment": {"state": "partial", "objective": "In a real-world comparison (107 versus 1.2×106 intervals), our library runs overlap queries 6.5×, nearest queries 15.5×, count_overlaps queries 38×, and coverage queries 15× faster than Bioframe.", "steps": [{"id": "read_published_estimate", "order": 1, "name": "Read the published estimate from the source", "kind": "other", "state": "complete", "blocked_on": null, "description": "The claim, method and evidence span published with the source article.", "duration_hours": null, "parameters": {}, "inputs": ["source:pmid:41325273"], "outputs": ["the published estimate and its evidence"], "instruments": [], "reagents": []}, {"id": "obtain_source_data", "order": 2, "name": "Obtain the source data from the named repository", "kind": "data-preparation", "state": "blocked", "blocked_on": "data:accession_absent", "description": "The GEO series accession resolves and its sample accessions are listed. This lane did not download the dataset or read its licence.", "duration_hours": null, "parameters": {}, "inputs": [], "outputs": ["the source data with its repository provenance"], "instruments": [], "reagents": []}, {"id": "run_analysis", "order": 3, "name": "Run the published analysis", "kind": "analysis", "state": "complete", "blocked_on": null, "description": "Apply the paper's statistical method to the resolved series and compare the result with the published target.", "duration_hours": null, "parameters": {}, "inputs": [], "outputs": ["the replicated result and its comparison"], "instruments": [], "reagents": []}], "materials": [], "instruments": [], "reagents": [], "conditions": {}, "gaps": ["step:obtain_source_data blocked on data:accession_absent"]}, "data": {"state": "partial", "assets": [{"id": "source:pmid:41325273", "role": "manuscript", "format": "xml", "media_type": "application/xml", "uri": "https://doi.org/10.1093/bioinformatics/btaf640", "sha256": "sha256:45dedb1ffec5173528f0295be2f9f588e13627c62e22708cc532c10b9939b5f1", "bytes": 19656, "access": "open", "state": "resolved", "reader": null, "reader_status": "reader_required", "identifier": "10.1093/bioinformatics/btaf640", "version": null, "note": "Full article fetched from PMC; the checksum covers the rendered section body, not the original XML bytes.", "structure": {"pmcid": "PMC12701801", "source": "europe_pmc_fulltext", "coverage": "whole_body", "text_hash_scope": "rendered_section_body"}}], "gaps": []}, "analysis": {"state": "absent", "methods": [], "targets": [], "gaps": ["method", "operations"]}, "capability": {"state": "complete", "requirements": [{"capability_id": "capability:computational_analysis", "label": "Computational analysis platform", "assay_families": ["meta_analysis", "machine_learning", "survival_analysis", "statistical_regression", "pca", "molecular_docking", "network_analysis", "differential_expression", "gwas"], "availability_state": "owned", "tier_unlocked": "R1", "candidate": "Automated Assays Laravel application and simulated adapters"}], "gaps": []}, "readiness": {"state": "blocked", "tier": "R4", "blocking_axis": "data", "blocked_on": "data:accession_absent", "axes": {"method": {"tier": "R4", "expressible": false, "required_operations": [], "missing_operations": [], "basis": "The reader quoted the method the paper names (not named), but the corpus's method extractor could not map it onto a DSL operation, so the method is published but not expressible."}, "capability": {"tier": "R1", "capability_id": "capability:computational_analysis", "availability_state": "owned", "basis": "The computational analysis is owned. The physical bench the paper used is not, so a physical validation cannot be run here."}, "data": {"tier": "R4", "reachable": false, "accessions": [], "resolved_accessions": [], "statements": ["Links to all datasets used in this study are provided in Section 3.2 of the Supplementary Materials."], "basis": "The paper names no accession to its source data, so the data has to come from the authors."}}, "can_run": {"dsl_operations": [], "assets": ["source:pmid:41325273"], "steps": ["run_analysis"]}, "must_build": {"dsl_operations": [], "converters": [], "readers": [], "capabilities": [], "data_gaps": ["data:geo:None"]}, "gaps": ["data:accession_absent"]}, "review": {"state": "partial", "machine_critique": {"kind": "extraction_review", "reader": {"model": "MiniMax-M3.1-Flash", "provider": "minimax", "provider_family": "minimax", "read_at": "2026-11-01T20:06:57Z", "independence": "single_reader"}, "evidence_check": {"method": null, "availability": "verbatim"}, "calls": 5, "usage": {"prompt_tokens": 6376, "completion_tokens": 818, "cached_tokens": 264}}, "human_critique": null, "stages": [{"id": "extraction", "state": "complete", "reader": "MiniMax-M3.1-Flash", "provider": "minimax"}], "gaps": ["review.independent_reader", "review.machine_critique", "review.human_critique"]}, "execution": {"state": "absent", "status": "not_run", "mode": null, "runs": [], "gaps": ["execution.not_run"]}, "provenance": {"entities": [{"id": "pmid:41325273", "type": "paper", "label": "pmid:41325273"}, {"id": "source:pmid:41325273", "type": "manuscript", "label": "PMID 41325273 full text"}, {"id": "data:geo:None", "type": "dataset", "label": "GEO None"}], "activities": [{"id": "activity-reading-sci-2026-11-01", "type": "classification", "used": [{"id": "pmid:41325273", "type": "paper", "label": "pmid:41325273"}, {"id": "source:pmid:41325273", "type": "manuscript", "label": "PMID 41325273 full text"}, {"id": "data:geo:None", "type": "dataset", "label": "GEO None"}], "agent": "agent:corpus-assay-engine"}, {"id": "activity-read-41325273", "type": "extraction", "used": ["source:pmid:41325273"], "agent": "agent:minimax-reader"}, {"id": "activity-resolve-None", "type": "accession_resolution", "used": ["data:geo:None"], "agent": "agent:corpus-assay-engine"}], "agents": [{"id": "agent:corpus-assay-engine", "type": "software", "label": "Corpus assay engine"}, {"id": "agent:minimax-reader", "type": "software", "label": "MiniMax literature reader"}], "source_run_id": "reading-sci-2026-11-01", "dsl_catalogue_fingerprint": "343fb30756ba6ec3", "capability_table_fingerprint": "2cb3e42f047d2ecf", "snapshot_id": "ba34b49490d9bfaf"}, "lineage": {"parent_id": null, "parent_hash": null, "fork_of": null, "generation": 0}, "gaps": ["execution.not_run", "method", "operations", "review.human_critique", "review.independent_reader", "review.machine_critique", "step:obtain_source_data blocked on data:accession_absent"]}