{ "schema_version": 2, "project": "GenomicTouchstone", "scope_as_of": "2026-10-18", "description": "Task catalogue for the revised benchmark scope. Dataset settings include related sources, lengths and split protocols; they do not represent 103 independent datasets.", "editorial_basis": "Task descriptions and biological context are excerpted from the active revised manuscript's Results and Methods, with LaTeX formatting and citation markers removed. The public 2025 preprint predates additions to this catalogue.", "paper": { "title": "Genomic Touchstone: Benchmarking Genomic Language Models in the Context of the Central Dogma", "url": "https://doi.org/10.1101/2025.06.25.661622", "type": "bioRxiv preprint", "year": 2025 }, "counts": { "task_types": 44, "dataset_settings": 103, "main_settings": 100, "long_range_settings": 3, "settings_by_modality": { "DNA": 55, "RNA": 33, "Protein": 15 } }, "tasks": [ { "name": "BRCA1/2 SNV Functional Impact Classification", "modality": "DNA", "category": "Genetic Variant Effect Prediction", "settings": [ { "id": "brca1_1024_coding", "objective": "Classification", "scope": "main" }, { "id": "brca1_1024_noncoding", "objective": "Classification", "scope": "main" }, { "id": "brca2_1024_coding", "objective": "Classification", "scope": "main" }, { "id": "brca2_1024_noncoding", "objective": "Classification", "scope": "main" } ], "description": "BRCA1/2 SNV functional impact classification uses a dataset that assays the functional consequences of single-nucleotide variants across coding and noncoding regions of the BRCA1 and BRCA2 genes.", "significance": "These genes play central roles in DNA repair and maintenance of genomic stability, and variant impact is directly relevant to hereditary cancer risk.", "methods": "Saturation genome editing provides four binary settings: coding and noncoding variants for each gene. Inputs are paired 1,024-bp reference and alternative sequences. BRCA1 separates loss of function from functional/intermediate variants; BRCA2 separates evidence supporting pathogenicity from benignity, excluding uncertain variants. Coordinate-based groups and a 1,024-bp buffer separate input windows across splits." }, { "name": "Genetic Disease Classification", "modality": "DNA", "category": "Genetic Variant Effect Prediction", "settings": [ { "id": "breast_disease_type_classification_512", "objective": "Classification", "scope": "main" }, { "id": "cardiovascular_disorders_classification_512", "objective": "Classification", "scope": "main" }, { "id": "kidney_disease_type_classification_512", "objective": "Classification", "scope": "main" } ], "description": "Genetic disease classification assesses whether models can predict disease categories associated with genetic variants across breast, cardiovascular and kidney disease datasets. We use paired 512-bp reference and alternative allele sequences for ClinVar SNVs, with gene-disjoint splits to evaluate generalization to previously unseen genes.", "significance": "Incorporating functional annotations and pathway information may help models develop more transferable representations that link genetic variation to disease-relevant biology.", "methods": "ClinVar single-nucleotide variants are represented by paired 512-bp reference and alternative sequences. The breast, cardiovascular and kidney settings contain four, six and three classes, respectively. Variants sharing genes or overlapping input windows are kept in the same split." }, { "name": "Pathogenicity Classification", "modality": "DNA", "category": "Genetic Variant Effect Prediction", "settings": [ { "id": "pathogenicity_classification_512", "objective": "Classification", "scope": "main" } ], "description": "Pathogenicity classification evaluates the ability of models to assign genetic variants to four ClinVar-derived categories (benign, likely benign, pathogenic, and likely pathogenic), providing a direct test of variant-level pathogenicity prediction. We use gene-disjoint splits to assess generalization to previously unseen genes.", "significance": "Genetic variant effect prediction tasks evaluate whether models can map single-nucleotide changes and their local or long-range genomic context to clinically and functionally meaningful labels.", "methods": "The four-class task uses ClinVar single-nucleotide variants with at least one review star and paired 512-bp reference and alternative inputs. Splits are gene-disjoint, with overlapping variant windows grouped together." }, { "name": "Splice Variant Effect Prediction", "modality": "DNA", "category": "Genetic Variant Effect Prediction", "settings": [ { "id": "splicevardb_1024_exonic", "objective": "Classification", "scope": "main" }, { "id": "splicevardb_1024_intronic", "objective": "Classification", "scope": "main" } ], "description": "Splice variant effect prediction evaluates the ability of models to distinguish splice-altering from normal variants in exonic and intronic regions using the SpliceVarDB dataset.", "significance": "These variants can perturb mRNA splicing patterns and thereby alter transcript structure and protein function, contributing to a wide range of genetic disorders and complex diseases.", "methods": "Two binary settings use experimentally validated SpliceVarDB labels and paired 1,024-bp reference and alternative sequences. Shared genes and overlapping windows are grouped jointly across the exonic and intronic settings. The main benchmark's AUPRC convention treats normal variants as the positive class." }, { "name": "eQTL Prediction", "modality": "DNA", "category": "Genetic Variant Effect Prediction", "settings": [ { "id": "eqtl_whole_blood_dnalongbench", "objective": "Classification", "scope": "long_range" } ], "description": "Expression quantitative trait locus prediction assesses whether models can link genetic variation to differences in target-gene expression, an important route through which genetic variants can influence cellular function and disease susceptibility. We use human whole-blood data from GTEx v8 to distinguish variant–gene pairs with strong fine-mapping support from matched pairs with low support. Paired reference and alternative allele sequences span the variant and the target-gene transcription start site, providing long-range genomic context for prediction.", "significance": "Pretraining that integrates tissue-specific regulatory profiles with variant-level functional measurements may help models better connect sequence changes in regulatory elements with their effects on target-gene expression.", "methods": "GTEx v8 variant–gene pairs follow the DNALongBench protocol: positives have SuSiE posterior inclusion probability above 0.9 and matched negatives below 0.01. Paired reference and alternative intervals span the variant and target-gene TSS and are padded or truncated to 450,000 bp. Shared genes, loci and overlapping spans stay together. This is a separate long-range evaluation." }, { "name": "CpG Methylation Prediction (Classification)", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "cpg_methylation", "objective": "Classification", "scope": "main" } ], "description": "For classification, CpG sites with at least 90% methylated reads were labeled as methylated, while those with less than 10% were labeled as unmethylated; remaining sites were discarded.", "significance": "CpG methylation prediction focuses on predicting both the status and quantitative levels of CpG methylation, a central epigenetic mark that modulates gene expression without altering the underlying DNA sequence. This modification also plays key roles in embryonic development, genomic imprinting and X-chromosome inactivation.", "methods": "Seven ENCODE experiments supply binary labels for 512-bp, CpG-centred sequences. Sites must have at least 90% or less than 10% methylated reads in every experiment; intermediate sites are excluded. Chromosome-based partitions follow the BEND preprocessing protocol." }, { "name": "CpG Methylation Prediction (Regression)", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "cpg_methylation_regression", "objective": "Regression", "scope": "main" } ], "description": "For regression, ENCODE-reported methylation percentages were divided by 100 to obtain seven continuous targets in the range [0,1], retaining intermediate methylation levels without applying the classification thresholds.", "significance": "CpG methylation prediction focuses on predicting both the status and quantitative levels of CpG methylation, a central epigenetic mark that modulates gene expression without altering the underlying DNA sequence. This modification also plays key roles in embryonic development, genomic imprinting and X-chromosome inactivation.", "methods": "ENCODE methylation percentages become seven targets between 0 and 1 for each 512-bp, CpG-centred input. Coverage and genotype filters follow the classification protocol, but intermediate methylation values are retained. Chromosome partitions are shared with classification, with eligible sites sampled within each partition." }, { "name": "Enhancer Types Classification", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "enhancers_types", "objective": "Classification", "scope": "main" } ], "description": "Enhancer type classification focuses on distinguishing tissue-specific and tissue-invariant enhancers from non-enhancer regions. The dataset uses human enhancer elements from ENCODE SCREEN, with tissue-invariant labels defined using the enhancer vocabulary of Meuleman et al..", "significance": "Enhancers are regulatory DNA elements that can increase the transcriptional activity of target genes over long genomic distances, with tissue-specific enhancers driving cell-type–specific expression and tissue-invariant enhancers reflecting shared core regulatory programs.", "methods": "Human enhancer annotations come from ENCODE SCREEN, with tissue-invariant regions defined by the Meuleman enhancer vocabulary. The three-class task uses 400-bp genomic sequences and non-overlapping non-enhancer negatives. Held-out chromosomes are used for validation and testing." }, { "name": "Epigenetic Marks Prediction (Classification)", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "histone_annotation_1", "objective": "Classification", "scope": "main" } ], "description": "We formulated two prediction tasks: multilabel classification of histone marks and regression of quantitative histone enrichment. The classification task used ten binary labels to represent the histone marks associated with each sequence, with one or more positive labels per sample.", "significance": "Epigenetic mark prediction tests whether models can predict histone modification peak presence and quantitative ChIP-seq enrichment from primary DNA sequence, providing an indirect readout of regulatory state.", "methods": "ENCODE ChIP-seq peaks define ten binary labels for 1,024-bp windows centred on peak summits. Overlapping annotations can assign multiple marks to a window. Overlapping windows and windows separated by at most 1,024 bp are grouped before splitting." }, { "name": "Epigenetic Marks Prediction (Regression)", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "histone_annotation_regression", "objective": "Regression", "scope": "main" } ], "description": "For the regression task, continuous fold-change-over-control signals were obtained from matched ENCODE bigWig tracks for the same ten K562 histone marks. Each sample had ten continuous targets, calculated by taking the arithmetic mean signal across the full 1,024 bp input window for each mark and then applying the log(1+x) transformation.", "significance": "Epigenetic mark prediction tests whether models can predict histone modification peak presence and quantitative ChIP-seq enrichment from primary DNA sequence, providing an indirect readout of regulatory state.", "methods": "Matched ENCODE bigWig tracks provide ten targets: mean fold-change-over-control signal over each 1,024-bp window, transformed with log(1+x). Inputs and split assignments follow the classification setting; samples with incomplete signal coverage are excluded." }, { "name": "Promoter Annotation", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "core_promoter_annotation", "objective": "Classification", "scope": "main" }, { "id": "promoter_annotation", "objective": "Classification", "scope": "main" } ], "description": "Promoter annotation assesses the ability of models to distinguish promoter regions based on core functional elements. We used human promoter sequences from EPDnew.", "significance": "The task includes both full promoter annotation and a more focused core promoter formulation that targets the minimal sequence required for transcription initiation in a narrow window around the transcription start site (TSS) (-34 bp to +35 bp).", "methods": "EPDnew supplies human promoters. The 500-bp full-promoter task has six multilabel categories; the 70-bp core-promoter task has four mutually exclusive categories. Shuffled negatives remain with their parent promoters. Shared genes, TSSs and overlapping windows define groups, with shared splits across the two settings." }, { "name": "Regulation Element Prediction", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "regulation", "objective": "Classification", "scope": "main" }, { "id": "regulation_multilabel", "objective": "Classification", "scope": "main" } ], "description": "Regulatory element prediction aims to identify core regulatory elements in the genome, including promoters, enhancers, CTCF binding sites, and open chromatin regions, which together shape the gene regulatory landscape. The regulatory annotations were obtained from the Ensembl Regulatory Build.", "significance": "Genomic function annotation tasks assess whether models can identify promoters, enhancer types, splice sites and other regulatory elements, predict epigenetic states and quantitative signals, and classify sequences by species of origin directly from DNA sequence. Together, these tasks range from recognition of local sequence features to prediction of complex regulatory states, enabling us to assess basic predictive capability on well-characterized signals.", "methods": "Ensembl Regulatory Build annotations define labels for 1,024-bp windows. A multilabel setting retains overlapping element labels, while a four-class setting retains windows with a single positive label. Both settings share split assignments, keeping overlapping windows together." }, { "name": "Species Classification", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "species_classification_1024", "objective": "Classification", "scope": "main" }, { "id": "species_classification_128", "objective": "Classification", "scope": "main" }, { "id": "species_classification_2048", "objective": "Classification", "scope": "main" }, { "id": "species_classification_256", "objective": "Classification", "scope": "main" }, { "id": "species_classification_512", "objective": "Classification", "scope": "main" } ], "description": "Species classification evaluates whether models can identify the species of origin of genomic fragments based on their sequence features. We used genome assemblies for six mammalian species obtained from NCBI.", "significance": "This task is motivated by applications in comparative genomics, species identification, and ancient DNA analysis, where reliable species-level discrimination from sequence alone is essential.", "methods": "NCBI genome assemblies provide sequences from human, lemur, pig, mouse, gorilla and hippo. Six-class settings use 128, 256, 512, 1,024 and 2,048 bp windows centred on the same loci. All lengths share species labels and chromosome-based split assignments." }, { "name": "Splice Sites Prediction", "modality": "DNA", "category": "Genomic Function Annotation", "settings": [ { "id": "splice_sites_acceptor", "objective": "Classification", "scope": "main" }, { "id": "splice_sites_donor", "objective": "Classification", "scope": "main" }, { "id": "splice_sites_all", "objective": "Classification", "scope": "main" } ], "description": "Splice site prediction evaluates the recognition of donor and acceptor sites that define exon–intron boundaries in eukaryotic genes, a key determinant of alternative splicing and transcript diversity. The benchmark uses human splice-site annotations from GENCODE.", "significance": "We assess the ability of gLMs to identify splice junctions directly from genomic sequence.", "methods": "GENCODE human annotations define three settings on shared 512-bp windows: donor presence, acceptor presence and a multilabel donor/acceptor/none task. Donor and acceptor labels can co-occur within a window. Shared genes, splice sites and overlapping or nearby intervals are grouped before splitting." }, { "name": "Enhancer Activity Prediction", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "enhancer_activity_deepstarr_hct116", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_brd2", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_brd4", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_cdk7", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_cdk9", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_med14", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_p300cbp", "objective": "Regression", "scope": "main" }, { "id": "enhancer_activity_hct116_parental", "objective": "Regression", "scope": "main" } ], "description": "Enhancer activity prediction asks models to quantify the regulatory strength of enhancer regions from sequence, using quantitative readouts as a proxy for enhancer-driven transcriptional output. The benchmark uses enhancer activity measurements in human HCT116 cells from the DeepSTARR study and the STARR-seq experiments of Neumayr et al..", "significance": "Enhancers are regulatory DNA elements that can increase the transcriptional activity of target genes over long genomic distances, with tissue-specific enhancers driving cell-type–specific expression and tissue-invariant enhancers reflecting shared core regulatory programs.", "methods": "Eight regression settings combine DeepSTARR reporter RNA-to-DNA log2 fold changes with seven Neumayr STARR-seq conditions, including cofactor perturbations and a parental control. Inputs are 249-bp oligonucleotides or 1,199-bp genomic windows. DeepSTARR design families stay together; the seven Neumayr settings share sequences and splits." }, { "name": "Enhancer Promoter Interaction Prediction", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "enhancer_promoter_interaction_gm12878", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_hela_s3", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_huvec", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_imr90", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_k562", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_nhek", "objective": "Classification", "scope": "main" }, { "id": "enhancer_promoter_interaction_gasperini_k562", "objective": "Classification", "scope": "main" } ], "description": "Enhancer–promoter interaction prediction evaluates whether models can identify enhancer–promoter relationships from paired DNA sequences. We assess both physical contacts supported by Hi-C data across six human cell lines and functional regulatory links supported by CRISPRi perturbation experiments in K562 cells.", "significance": "Together, these regression and interaction tasks probe how well gLMs capture cis-regulatory grammar, long-range dependencies, and the combinatorial nature of regulatory control from sequence alone.", "methods": "Six binary settings use TargetFinder Hi-C labels, and a seventh uses Gasperini K562 CRISPRi labels. Inputs pair a 3,000-bp enhancer with a 2,000-bp promoter. Shared biological entities remain in one split; the functional setting also groups overlapping enhancer-to-TSS spans." }, { "name": "Enhancer Target Gene Prediction", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "enhancer_target_gene_gasperini_k562", "objective": "Classification", "scope": "long_range" } ], "description": "Enhancer target gene prediction tests whether models can identify functional enhancer–gene relationships using long-range genomic context. Inputs comprise continuous genomic sequences spanning the candidate enhancer and target-gene transcription start site, with labels derived from the K562 CRISPRi screen of Gasperini et al..", "significance": "Predicting these relationships requires connecting enhancer-associated regulatory signals with the transcriptional response of a specific target gene, taking into account enhancer–promoter compatibility and the surrounding genomic context.", "methods": "Gasperini K562 CRISPRi labels indicate significant target-gene downregulation after enhancer perturbation. DNALongBench-style inputs span enhancer and TSS with flanks, masking other annotated enhancers; lengths range from 6,000 to 449,407 bp. Shared genes, enhancers and overlapping intervals stay together. This is a separate long-range evaluation." }, { "name": "Gene Expression Prediction", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "bulk_rna_expression_1024", "objective": "Regression", "scope": "main" }, { "id": "bulk_rna_expression_128", "objective": "Regression", "scope": "main" }, { "id": "bulk_rna_expression_2048", "objective": "Regression", "scope": "main" }, { "id": "bulk_rna_expression_256", "objective": "Regression", "scope": "main" }, { "id": "bulk_rna_expression_512", "objective": "Regression", "scope": "main" } ], "description": "Gene expression prediction models the relationship between local genomic sequence and quantitative gene activity levels, providing a window into cis-regulatory logic that shapes cellular identity, development, and responses to environmental cues. We used the ExPecto expression dataset.", "significance": "Regulatory mechanism modeling tasks evaluate whether models can connect local and distal sequence features to quantitative and context-dependent regulatory readouts, including chromatin accessibility, enhancer output, enhancer–promoter contacts, gene expression, and transcription initiation signals. Together, these regression and interaction tasks probe how well gLMs capture cis-regulatory grammar, long-range dependencies, and the combinatorial nature of regulatory control from sequence alone.", "methods": "ExPecto provides 218 expression profiles across GTEx, Roadmap Epigenomics and ENCODE. Targets are log(1+x)-transformed and standardized using training-set statistics. Five input lengths from 128 to 2,048 bp share genes, targets and chromosome partitions." }, { "name": "Regulatory Activity Prediction", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "regulatory_activity_gm12878", "objective": "Regression", "scope": "main" }, { "id": "regulatory_activity_h1esc", "objective": "Regression", "scope": "main" }, { "id": "regulatory_activity_hepg2", "objective": "Regression", "scope": "main" }, { "id": "regulatory_activity_imr90", "objective": "Regression", "scope": "main" }, { "id": "regulatory_activity_k562", "objective": "Regression", "scope": "main" } ], "description": "Regulatory activity prediction centers on predicting quantitative chromatin accessibility from DNA sequence, using DNase-seq read counts as a proxy for open chromatin and regulatory activity. We followed DART-Eval in using processed ENCODE DNase-seq measurements from five human cell lines.", "significance": "Regulatory mechanism modeling tasks evaluate whether models can connect local and distal sequence features to quantitative and context-dependent regulatory readouts, including chromatin accessibility, enhancer output, enhancer–promoter contacts, gene expression, and transcription initiation signals. Together, these regression and interaction tasks probe how well gLMs capture cis-regulatory grammar, long-range dependencies, and the combinatorial nature of regulatory control from sequence alone.", "methods": "DART-Eval-processed ENCODE DNase-seq data define five regression settings. Each 2,114-bp peak or matched-background window predicts log(1+x), where x counts read endpoints in the central 1,000 bp. Splits follow the official DART-Eval chromosome partitions." }, { "name": "Transcription Initiation Signal Prediction (Long-range)", "modality": "DNA", "category": "Regulatory Mechanism Modeling", "settings": [ { "id": "transcription_initiation_signal_prediction", "objective": "Regression", "scope": "long_range" } ], "description": "Transcription initiation signal prediction assesses whether models can predict the positions and intensities of transcription initiation signals from long-range genomic sequences. We use 100-kb human DNA sequences to predict strand-specific profiles of FANTOM CAGE, ENCODE CAGE, ENCODE RAMPAGE, GRO-cap and PRO-cap signals.", "significance": "Modeling transcription at this scale involves resolving where initiation occurs and how its intensity varies along extended DNA sequences. Further progress in long-range genomic modeling may therefore benefit from representations that preserve fine-scale regulatory signals while integrating broader genomic context.", "methods": "DNALongBench preprocessing supplies 100,000-bp human inputs with ten channels covering FANTOM CAGE, ENCODE CAGE, ENCODE RAMPAGE, GRO-cap and PRO-cap on both strands. Chromosome 10 is held out for validation and chromosomes 8 and 9 for testing. This separate long-range evaluation reports Pearson correlation." }, { "name": "APA Isoform Prediction", "modality": "RNA", "category": "Post-transcriptional Regulation", "settings": [ { "id": "isoform", "objective": "Regression", "scope": "main" } ], "description": "APA isoform prediction benchmarks the performance of models in predicting the usage of proximal polyadenylation signals (PAS) in the 3' UTR, a key mechanism by which alternative polyadenylation generates transcript isoform diversity and modulates mRNA stability and translation.", "significance": "Polyadenylation-site choice depends on the core PAS together with surrounding regulatory elements and their relative positions.", "methods": "APARENT reporter measurements, processed through BEACON, provide proximal PAS usage targets for 186-nucleotide windows. The main setting holds out reporter design families: TOMM5, DoubleDope, ATR and SOX13 for training, AARS for validation and Simple for testing." }, { "name": "Modification Prediction", "modality": "RNA", "category": "Post-transcriptional Regulation", "settings": [ { "id": "modification", "objective": "Classification", "scope": "main" } ], "description": "Modification prediction assesses the capacity of models to identify the types of RNA modifications present on precursor mRNA (pre-mRNA) sequences. The RNA modification data were compiled by Song et al. from GEO, RMBase and RADAR.", "significance": "Chemical modifications such as m⁶A, m¹A, and m⁵C expand the structural and functional diversity of RNA and regulate multiple stages of the RNA life cycle.", "methods": "Data compiled by Song et al. from GEO, RMBase and RADAR define 12 modification labels for 101-nucleotide windows. Positive sites linked by candidate genes, sites or overlapping intervals are grouped before splitting. Same-gene background windows receive all-zero labels and stay with their paired positives." }, { "name": "SARS-CoV-2 Vaccine Degradation Prediction", "modality": "RNA", "category": "Post-transcriptional Regulation", "settings": [ { "id": "cov_vaccine_degradation", "objective": "Regression", "scope": "main" }, { "id": "cov_vaccine_degradation_random_iid", "objective": "Regression", "scope": "main" } ], "description": "SARS-CoV-2 vaccine degradation prediction investigates the capability of models to predict in-line hydrolysis rates for mRNA sequences used in SARS-CoV-2 vaccine designs, using In-line-seq measurements over the first 68 nucleotides as quantitative readouts of degradation.", "significance": "RNA hydrolysis is sensitive to local structural flexibility and backbone geometry, properties that general sequence pretraining may not adequately capture.", "methods": "The sequence-level target averages the magnesium-condition degradation signal at 50 °C over the first 68 measured nucleotides and is standardized using training statistics. Inputs are 81-nucleotide windows. Random and predicted-structure-grouped settings are evaluated separately." }, { "name": "Programmable RNA Switches Prediction", "modality": "RNA", "category": "RNA Engineering Applications", "settings": [ { "id": "programmable_rna_switches", "objective": "Regression", "scope": "main" } ], "description": "Programmable RNA switch prediction evaluates whether models can predict the functional performance of engineered RNA switches from sequence, a key design problem in RNA-based sensing and synthetic biology.", "significance": "RNA engineering applications tasks use synthetic and engineered constructs to test how far gLMs can support design problems in RNA-based sensing, regulation, and expression control. They cover programmable RNA switches, ligand-responsive riboswitches, and codon-level tuning of protein output, linking model predictions directly to sequence choices that determine the behavior of engineered RNAs.", "methods": "An E. coli toehold-switch library provides three normalized activity targets measured by fluorescence sorting and sequencing. Inputs are 148-nucleotide constructs containing trigger, switch and flanks. Source pathogens or human target genes define split groups; the random-trigger library forms one source group." }, { "name": "Tc-Riboswitches Prediction", "modality": "RNA", "category": "RNA Engineering Applications", "settings": [ { "id": "tc_riboswitches", "objective": "Regression", "scope": "main" }, { "id": "tc_riboswitches_random_iid", "objective": "Regression", "scope": "main" } ], "description": "Tc-riboswitch prediction measures the efficacy of models in predicting the regulatory performance of tetracycline (Tc) riboswitch variants from sequence, a representative test of modeling synthetic, structured RNA elements used in tunable gene control.", "significance": "Tc-riboswitch activity depends on the coupling of tetracycline binding, RNA folding and translational regulation.", "methods": "Yeast GFP reporter measurements provide the log switching factor: expression without tetracycline divided by expression with tetracycline. Inputs span 66–75 nucleotides. Separate settings use a random split or a chronological split with rounds 1–2 for training, round 3 for validation and round 4 for testing." }, { "name": "mRFP Expression Prediction", "modality": "RNA", "category": "RNA Engineering Applications", "settings": [ { "id": "mrfp_expression", "objective": "Regression", "scope": "main" } ], "description": "mRFP expression prediction tests whether models can predict protein production levels from mRNA sequences that encode the same amino acid sequence but differ in synonymous codon usage.", "significance": "This task directly probes a model's ability to capture codon optimality effects on translation efficiency in a synthetic setting.", "methods": "Three E. coli codon libraries provide complete 678-nucleotide mRFP CDS inputs. The target is log mean fluorescence normalized to a GFPuv reference. Identical CDSs stay in one split while library membership and expression deciles are balanced." }, { "name": "Expression Level Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "hek_el", "objective": "Regression", "scope": "main" }, { "id": "muscle_el", "objective": "Regression", "scope": "main" }, { "id": "pc3_el", "objective": "Regression", "scope": "main" } ], "description": "Expression level prediction assesses whether models can relate 5' UTR sequence features to cellular mRNA abundance.", "significance": "Transcript abundance reflects the balance between RNA synthesis and degradation and determines the pool of mRNAs available for protein production.", "methods": "UTR-LM-processed measurements from HEK293T, PC3 and muscle define three regression settings using log RNA-seq RPKM. Inputs are 45–100 nucleotides immediately upstream of the CDS. Splits keep shared genes and identical or reverse-complement UTRs together." }, { "name": "Mean Ribosome Loading Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "mean_ribosome_loading", "objective": "Regression", "scope": "main" }, { "id": "mean_ribosome_loading_human", "objective": "Regression", "scope": "main" } ], "description": "Mean ribosome loading prediction models translation efficiency from sequence by estimating ribosome loading on mRNAs, with a particular focus on 5' UTR sequences.", "significance": "RNA functional studies tasks investigate how sequence features in 5' UTRs, coding regions, and non-coding RNAs relate to downstream molecular phenotypes, including translation efficiency, transcript stability, transcript and protein abundance, RNA subcellular localization, and ncRNA function, providing a suite of tests for how well gLMs connect RNA sequence to expression- and translation-related outcomes.", "methods": "Massively parallel translation assays provide standardized continuous loading targets for 25–100 nucleotide UTRs. One setting trains on random UTRs and tests on human-derived UTRs; the other uses human-only data grouped by available gene, design and sequence-similarity information." }, { "name": "Non-coding RNA Classification", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "noncodingrnafamily", "objective": "Classification", "scope": "main" } ], "description": "Following the previous study, ncRNA sequences were obtained from Rfam release 12. Each sequence was assigned to one of 13 functional classes: miRNA, 5S rRNA, 5.8S rRNA, ribozymes, C/D-box, H/ACA-box, scaRNA, tRNA, Group I introns, Group II introns, IRES, leader, and riboswitch.", "significance": "RNA functional studies tasks investigate how sequence features in 5' UTRs, coding regions, and non-coding RNAs relate to downstream molecular phenotypes, including translation efficiency, transcript stability, transcript and protein abundance, RNA subcellular localization, and ncRNA function, providing a suite of tests for how well gLMs connect RNA sequence to expression- and translation-related outcomes.", "methods": "Rfam-derived inputs range from 38 to 1,182 nucleotides and cover classes including miRNA, ribosomal RNA, tRNA, ribozymes and riboswitches. GraphPart partitions sequences at an 80% global nucleotide identity threshold, retaining all 13 classes in each split." }, { "name": "RNA Localization Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "rna_localization_rnagps_apexseq", "objective": "Classification", "scope": "main" } ], "description": "RNA subcellular localization. The RNA-GPS/APEX-seq task evaluates prediction of the subcellular distribution of mature transcripts in human HEK293T cells.", "significance": "RNA functional studies tasks investigate how sequence features in 5' UTRs, coding regions, and non-coding RNAs relate to downstream molecular phenotypes, including translation efficiency, transcript stability, transcript and protein abundance, RNA subcellular localization, and ncRNA function, providing a suite of tests for how well gLMs connect RNA sequence to expression- and translation-related outcomes.", "methods": "RNA-GPS/APEX-seq measurements in HEK293T cells define eight enrichment labels. Inputs retain a 3′-anchored transcript window within each model's length limit. Gene identity, identical or reverse-complement sequences, and nucleotide similarity groups determine splits." }, { "name": "Transcript and Protein Abundance Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "pa_athaliana", "objective": "Regression", "scope": "main" }, { "id": "pa_dmelanogaster", "objective": "Regression", "scope": "main" }, { "id": "pa_ecoli", "objective": "Regression", "scope": "main" }, { "id": "pa_hsapiens", "objective": "Regression", "scope": "main" }, { "id": "pa_scerevisiae", "objective": "Regression", "scope": "main" }, { "id": "ta_athaliana", "objective": "Regression", "scope": "main" }, { "id": "ta_dmelanogaster", "objective": "Regression", "scope": "main" }, { "id": "ta_ecoli", "objective": "Regression", "scope": "main" }, { "id": "ta_hsapiens", "objective": "Regression", "scope": "main" }, { "id": "ta_hvolcanii", "objective": "Regression", "scope": "main" }, { "id": "ta_ppastoris", "objective": "Regression", "scope": "main" }, { "id": "ta_scerevisiae", "objective": "Regression", "scope": "main" } ], "description": "Transcript and protein abundance prediction evaluates transcript abundance (TA) and protein abundance (PA) as separate regression targets, using measurements from transcriptomics and proteomics experiments as quantitative readouts of gene expression at the RNA and protein levels. The CaLM datasets combine transcript abundance measurements from GEO, Expression Atlas, SRA and published human tissue data, with protein abundance measurements from PaxDb.", "significance": "Coding sequences contribute to gene-expression regulation through links between codon usage, mRNA stability and translation, in addition to specifying amino-acid sequences.", "methods": "CaLM datasets provide seven transcript-abundance and five protein-abundance settings. Targets use source log-transformed TPM for transcripts and log(1+x) for proteins. Within species, shared identifiers and CDS or protein similarity define groups, with shared assignments across the two abundance types." }, { "name": "Translation Efficiency Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "hek_te", "objective": "Regression", "scope": "main" }, { "id": "muscle_te", "objective": "Regression", "scope": "main" }, { "id": "pc3_te", "objective": "Regression", "scope": "main" }, { "id": "translation_efficiency_ribonn_human_mean", "objective": "Regression", "scope": "main" } ], "description": "Translation efficiency prediction assesses whether models can connect RNA sequence features with the efficiency of protein synthesis from mRNA.", "significance": "Translational regulation involves ribosome recruitment and initiation within the 5' UTR, together with codon-dependent ribosome progression through the coding sequence.", "methods": "Three human UTR settings use log Ribo-seq-to-RNA-seq RPKM ratios from HEK, PC3 and muscle, sharing gene-disjoint splits with expression-level prediction. A fourth RiboNN setting uses full-transcript sequences and mean processed TE across available cellular contexts, following the authors' outer-fold-0 test assignment." }, { "name": "mRNA Stability Prediction", "modality": "RNA", "category": "RNA Functional Studies", "settings": [ { "id": "mrna_stability", "objective": "Regression", "scope": "main" }, { "id": "mrna_half_life_saluki_human", "objective": "Regression", "scope": "main" } ], "description": "mRNA stability prediction evaluates whether models can capture sequence features that influence transcript stability, a key determinant of steady-state gene expression and protein abundance.", "significance": "Coding sequences contribute to gene-expression regulation through links between codon usage, mRNA stability and translation, in addition to specifying amino-acid sequences.", "methods": "iCodon supplies profile-standardized decay rates for CDSs from human, mouse, frog and fish, with gene-disjoint splits. A separate Saluki human setting uses spliced transcripts and the authors' processed consensus half-life score, with their published fold assignments." }, { "name": "Enzyme Classification", "modality": "Protein", "category": "Protein Functional Annotation", "settings": [ { "id": "enzyme", "objective": "Classification", "scope": "main" } ], "description": "UniProt enzyme annotations linked via Ensembl protein IDs were used to retrieve and label proteins at the Enzyme Commission (EC) first-digit level. Proteins were assigned to four classes: oxidoreductases (EC 1), transferases (EC 2), hydrolases (EC 3), and other enzymes (EC 4–7).", "significance": "Enzyme classification serves as an effective evaluation of a gLM’s capacity to detect catalytic classes such as oxidoreductases, transferases, hydrolases, lyases, isomerases, and ligases based on sequence features that encode enzymatic function.", "methods": "UniProt EC annotations define oxidoreductases, transferases, hydrolases and other enzymes (EC 4–7). Incomplete or conflicting first-digit assignments are excluded. Verified CDS–protein pairs share labels and splits, with shared genes and protein homology groups kept together." }, { "name": "Post-translational Modifications Prediction", "modality": "Protein", "category": "Protein Functional Annotation", "settings": [ { "id": "ptm", "objective": "Classification", "scope": "main" } ], "description": "Post-translational modifications prediction tests the capability of gLMs to predict protein-level post-translational modification (PTM) classes from the genomic sequences encoding proteins, probing their ability to capture sequence determinants of protein-level regulation. We constructed the dataset from protein sequences and PTM annotations in UniProt.", "significance": "Together, these benchmarks move beyond structural attributes to focus on functional consequences of sequence variation and composition, offering a complementary view of how genomic and protein language models encode information about protein activity and regulation.", "methods": "UniProt modification annotations are deduplicated and summarized per protein. Phosphorylation or glycosylation defines the class when it accounts for more than half of annotations; other proteins form the other/mixed class. Paired CDS and protein inputs share gene- and homology-grouped splits." }, { "name": "Protein Domain Classification", "modality": "Protein", "category": "Protein Functional Annotation", "settings": [ { "id": "domain", "objective": "Classification", "scope": "main" } ], "description": "Protein domain classification assesses whether models can distinguish proteins by their domain families, which contribute to functions such as catalysis, nucleic-acid binding and protein interactions. We used UniProt annotations to evaluate four domain families: Protein kinase, BTB, RNA recognition motif (RRM), and basic helix–loop–helix (bHLH).", "significance": "Together, these benchmarks move beyond structural attributes to focus on functional consequences of sequence variation and composition, offering a complementary view of how genomic and protein language models encode information about protein activity and regulation.", "methods": "The task retains proteins with exactly one annotated UniProt domain in the Protein kinase, BTB, RNA recognition motif or basic helix–loop–helix families. Verified CDS–protein pairs use shared labels and splits, keeping shared genes and protein homology groups together." }, { "name": "Protein Subcellular Localization (DeepLoc2)", "modality": "Protein", "category": "Protein Functional Annotation", "settings": [ { "id": "protein_localization_deeploc2", "objective": "Classification", "scope": "main" } ], "description": "Protein subcellular localization assesses whether models can predict the cellular compartments in which proteins function from sequence information. We evaluated multilabel prediction across ten compartments using a CDS-paired subset of the Swiss-Prot dataset released with DeepLoc 2.0.", "significance": "Signal peptides and other position-dependent targeting sequences that guide proteins to different cellular compartments may be more readily learned through amino-acid sequence pretraining.", "methods": "A CDS-paired subset of DeepLoc 2.0 Swiss-Prot data provides ten localization labels. CDS translations are verified against full protein sequences, and CDSs up to 4,096 nucleotides are retained. Official partitions 0–2, 3 and 4 define training, validation and testing, respectively." }, { "name": "Protein Variant Classification", "modality": "Protein", "category": "Protein Functional Annotation", "settings": [ { "id": "variant", "objective": "Classification", "scope": "main" } ], "description": "Protein variant classification assesses whether gLMs can distinguish proteins with multiple amino-acid–altering variant annotations from other annotated proteins using their reference coding sequences. Protein sequences and variant annotations were obtained from UniProt.", "significance": "Together, these benchmarks move beyond structural attributes to focus on functional consequences of sequence variation and composition, offering a complementary view of how genomic and protein language models encode information about protein activity and regulation.", "methods": "UniProt annotations define an enriched class when a protein has at least two amino-acid-changing annotations and more than twice as many changing as unchanged annotations; other eligible proteins form the mixed class. Full-length reference CDS–protein pairs share gene- and homology-grouped splits, subject to model input limits." }, { "name": "Beta-Lactamase Activity Prediction", "modality": "Protein", "category": "Protein Property Prediction", "settings": [ { "id": "beta_lactamase_complete", "objective": "Regression", "scope": "main" }, { "id": "beta_lactamase_unique", "objective": "Regression", "scope": "main" } ], "description": "Beta-lactamase activity prediction evaluates a model's capacity to predict the fitness effects of single-codon substitutions in the gene encoding TEM-1 β-lactamase, an enzyme conferring ampicillin resistance. Fitness is measured from deep mutational scanning experiments as a function of growth across increasing ampicillin concentrations. Following the dataset definitions in True-CDS, we evaluated the Complete dataset using CDS inputs and the Unique dataset, containing one CDS per distinct protein sequence, using both CDS and protein inputs.", "significance": "Protein property prediction tasks connect sequence variation to quantitative biophysical traits, including enzymatic activity, fluorescence, proteolysis-derived stability, and melting temperature, using data from mutational screens, proteolysis assays and thermal profiling. Together, these benchmarks probe how well models capture fine-scale sequence to property relationships in both broad and wild type proximal mutational regimes, providing a stringent evaluation of sequence based generalization for both genomic and protein language models.", "methods": "Deep mutational scanning yields wild-type-normalized fitness across ampicillin concentrations. The Complete setting retains distinct codon variants for CDS prediction; Unique provides one CDS per protein sequence for paired views. Released test assignments are retained, with position-specific amino-acid effects grouped when reconstructing Complete training and validation sets." }, { "name": "Fluorescence Prediction", "modality": "Protein", "category": "Protein Property Prediction", "settings": [ { "id": "fluorescence_amacgfp", "objective": "Regression", "scope": "main" }, { "id": "fluorescence_avgfp", "objective": "Regression", "scope": "main" }, { "id": "fluorescence_cgregfp", "objective": "Regression", "scope": "main" }, { "id": "fluorescence_pplugfp2", "objective": "Regression", "scope": "main" } ], "description": "Fluorescence prediction evaluates whether models can predict log-fluorescence values for fluorescent protein variants carrying multiple mutations, probing their ability to capture how combinations of mutations shape protein function. We used the avGFP measurements of Sarkisyan et al. and the amacGFP, cgreGFP and ppluGFP2 measurements of Gonzalez Somermeyer et al..", "significance": "Protein property prediction tasks connect sequence variation to quantitative biophysical traits, including enzymatic activity, fluorescence, proteolysis-derived stability, and melting temperature, using data from mutational screens, proteolysis assays and thermal profiling. Together, these benchmarks probe how well models capture fine-scale sequence to property relationships in both broad and wild type proximal mutational regimes, providing a stringent evaluation of sequence based generalization for both genomic and protein language models.", "methods": "Four regression settings use avGFP, amacGFP, cgreGFP and ppluGFP2 measurements with verified CDS–protein pairs and log-fluorescence targets. Training and validation contain at most three amino-acid substitutions from the parent; testing contains four or more." }, { "name": "Melting Point Prediction", "modality": "Protein", "category": "Protein Property Prediction", "settings": [ { "id": "melting_point", "objective": "Regression", "scope": "main" } ], "description": "Melting point prediction focuses on protein thermal stability, asking models to recover experimental melting temperatures from sequence alone. We used experimental melting temperatures from the Meltome atlas.", "significance": "Protein property prediction tasks connect sequence variation to quantitative biophysical traits, including enzymatic activity, fluorescence, proteolysis-derived stability, and melting temperature, using data from mutational screens, proteolysis assays and thermal profiling. Together, these benchmarks probe how well models capture fine-scale sequence to property relationships in both broad and wild type proximal mutational regimes, providing a stringent evaluation of sequence based generalization for both genomic and protein language models.", "methods": "Meltome thermal proteome profiling provides melting temperatures in degrees Celsius. True-CDS coding sequences are paired with source-validated protein sequences. The FLIP mixed-benchmark test assignment is retained, and published sequence clusters stay together when reconstructing training and validation." }, { "name": "Protein Stability Prediction", "modality": "Protein", "category": "Protein Property Prediction", "settings": [ { "id": "stability", "objective": "Regression", "scope": "main" } ], "description": "Protein stability prediction tests how well models can predict sequence-dependent differences in protein stability, a key constraint in protein engineering where even small stability shifts can strongly affect function. We used experimental stability measurements from Rocklin et al., as curated in TAPE, with the chymotrypsin-derived stability score as the regression target.", "significance": "Protein property prediction tasks connect sequence variation to quantitative biophysical traits, including enzymatic activity, fluorescence, proteolysis-derived stability, and melting temperature, using data from mutational screens, proteolysis assays and thermal profiling. Together, these benchmarks probe how well models capture fine-scale sequence to property relationships in both broad and wild type proximal mutational regimes, providing a stringent evaluation of sequence based generalization for both genomic and protein language models.", "methods": "Rocklin measurements curated in TAPE provide chymotrypsin-derived stability scores for short designed proteins. Verified CDS–protein pairs share parent-landscape groups and the released test assignment. CDS and amino-acid similarity filtering applies the same test exclusions to both views." }, { "name": "Secondary Structure Analysis", "modality": "Protein", "category": "Structural Analysis and Prediction", "settings": [ { "id": "structure", "objective": "Classification", "scope": "main" } ], "description": "Secondary structure analysis evaluates whether gLMs can predict protein-level secondary-structure classes from coding sequences, probing their ability to capture structural characteristics relevant to protein architecture and function. The benchmark was constructed from protein sequences and secondary-structure annotations in UniProt.", "significance": "By comparing gLMs with protein language models in these cross-modality settings, we assess how much information about folding and membrane topology is already encoded in genomic representations and which architectures translate codon-level context most effectively into structural constraints.", "methods": "UniProt helix and strand segment annotations define three classes: a class is dominant when its segment count exceeds 60% of the combined count; otherwise the protein is mixed. Paired CDS and protein inputs share gene- and homology-grouped splits, subject to model length limits." }, { "name": "Transmembrane Region Prediction", "modality": "Protein", "category": "Structural Analysis and Prediction", "settings": [ { "id": "transmembrane", "objective": "Classification", "scope": "main" } ], "description": "Transmembrane region prediction assesses the ability of gLMs to classify proteins by the number of membrane-spanning segments using coding DNA sequences, probing their ability to capture biophysically constrained sequence signals that underlie membrane embedding. The dataset uses protein sequences and transmembrane annotations from UniProt.", "significance": "By comparing gLMs with protein language models in these cross-modality settings, we assess how much information about folding and membrane topology is already encoded in genomic representations and which architectures translate codon-level context most effectively into structural constraints.", "methods": "UniProt transmembrane annotations define six classes: 1, 2, 3–4, 5–6, 7–8 or at least 9 segments. Full-length CDS–protein pairs share labels and split assignments, with shared genes and protein homology groups kept together and inputs limited by each model." } ] }