Running INFOG and GDR on one million cells

From the previous release

This page was recovered from the PIASO v1.1.0 documentation and has not been re-run against the current release. The code is preserved as published; some calls may have changed. The pages under All tutorials are the ones executed against this release.

The original v1.1.0 site is archived at https://genecell.github.io/PIASO/.

Running INFOG and GDR on one million cells

import numpy as np
import pandas as pd
import scanpy as sc
sc.set_figure_params(dpi=80,dpi_save=300, color_map='viridis',facecolor='white')
from matplotlib import rcParams
# To modify the default figure size, use rcParams.
rcParams['figure.figsize'] = 4, 4
rcParams['font.sans-serif'] = "Arial"
rcParams['font.family'] = "Arial"
sc.settings.verbosity = 3
sc.logging.print_header()

stderr

/tmp/ipykernel_884065/1353975569.py:11: RuntimeWarning: Failed to import dependencies for application/vnd.jupyter.widget-view+json representation. (ModuleNotFoundError: No module named 'ipywidgets')
  sc.logging.print_header()

| Dependency        | Version     |
| ----------------- | ----------- |
| cycler            | 0.12.1      |
| matplotlib-inline | 0.1.7       |
| jedi              | 0.19.2      |
| natsort           | 8.4.0       |
| ipython           | 8.37.0      |
| pillow            | 11.2.1      |
| executing         | 2.2.0       |
| leidenalg         | 0.10.2      |
| llvmlite          | 0.44.0      |
| pytz              | 2025.2      |
| joblib            | 1.5.1       |
| prompt_toolkit    | 3.0.51      |
| decorator         | 5.2.1       |
| stack-data        | 0.6.3       |
| wcwidth           | 0.2.13      |
| igraph            | 0.11.9      |
| parso             | 0.8.4       |
| debugpy           | 1.8.14      |
| python-dateutil   | 2.9.0.post0 |
| six               | 1.17.0      |
| texttable         | 1.7.0       |
| psutil            | 7.0.0       |
| asttokens         | 3.0.0       |
... [15 more lines]

save_dir='.../Result/single-cell/Methods/DataProcessing/AsianImmuneDiversityAtlasPhase1v2'
sc.settings.figdir = save_dir
prefix='AsianImmuneDiversityAtlasPhase1v2'
import os
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

sc.set_figure_params(dpi=80,dpi_save=300, color_map='viridis',facecolor='white')
rcParams['figure.figsize'] = 4, 4

adata=sc.read('.../Result/single-cell/Methods/DataProcessing/AsianImmuneDiversityAtlasPhase1v2/AsianImmuneDiversityAtlasPhase1v2.h5ad')

adata

AnnData object with n_obs × n_vars = 1265624 × 35477
    obs: 'reference_genome', 'gene_annotation_version', 'alignment_software', 'intronic_reads_counted', 'library_id', 'assay_ontology_term_id', 'sequenced_fragment', 'cell_number_loaded', 'institute', 'is_primary_data', 'cell_type_ontology_term_id', 'author_cell_type', 'sample_id', 'sample_preservation_method', 'tissue_ontology_term_id', 'development_stage_ontology_term_id', 'sample_collection_method', 'donor_BMI_at_collection', 'tissue_type', 'suspension_derivation_process', 'suspension_enriched_cell_types', 'cell_viability_percentage', 'suspension_uuid', 'suspension_type', 'donor_id', 'self_reported_ethnicity_ontology_term_id', 'donor_living_at_sample_collection', 'disease_ontology_term_id', 'sex_ontology_term_id', 'nCount_RNA', 'nFeature_RNA', 'pMito', 'NODG', 'nUMI', 'Country', 'Annotation_Level1', 'Annotation_Level2', 'Annotation_Level3', 'Annotation_Level4', 'Smoking Status', 'cell_type', 'assay', 'disease', 'sex', 'tissue', 'self_reported_ethnicity', 'development_stage', 'observation_joinid'
    var: 'feature_is_filtered', 'feature_name', 'feature_reference', 'feature_biotype', 'feature_length', 'feature_type'
    uns: 'citation', 'default_embedding', 'organism', 'organism_ontology_term_id', 'schema_reference', 'schema_version', 'title'
    obsm: 'X_umap'

adata.var.head()

adata.var['Ensemble_ID']=adata.var_names.copy()

adata.var_names=adata.var['feature_name'].values.astype(str).copy()

adata.var.head()

adata.var.tail(30)

adata.var_names=[np.str_.split(i, '_ENSG')[0] for i in adata.var_names]

adata.var_names_make_unique()

adata.var.head()

adata.raw.X.data

array([1., 1., 1., ..., 2., 1., 1.], shape=(2545339623,), dtype=float32)

adata.layers['raw']=adata.raw.X.copy()

del adata.raw

adata.X.data

array([1.21232039, 1.21232039, 1.21232039, ..., 2.99794589, 1.82356588,
       3.96914141], shape=(2544339517,))

adata

AnnData object with n_obs × n_vars = 1265624 × 35477
    obs: 'reference_genome', 'gene_annotation_version', 'alignment_software', 'intronic_reads_counted', 'library_id', 'assay_ontology_term_id', 'sequenced_fragment', 'cell_number_loaded', 'institute', 'is_primary_data', 'cell_type_ontology_term_id', 'author_cell_type', 'sample_id', 'sample_preservation_method', 'tissue_ontology_term_id', 'development_stage_ontology_term_id', 'sample_collection_method', 'donor_BMI_at_collection', 'tissue_type', 'suspension_derivation_process', 'suspension_enriched_cell_types', 'cell_viability_percentage', 'suspension_uuid', 'suspension_type', 'donor_id', 'self_reported_ethnicity_ontology_term_id', 'donor_living_at_sample_collection', 'disease_ontology_term_id', 'sex_ontology_term_id', 'nCount_RNA', 'nFeature_RNA', 'pMito', 'NODG', 'nUMI', 'Country', 'Annotation_Level1', 'Annotation_Level2', 'Annotation_Level3', 'Annotation_Level4', 'Smoking Status', 'cell_type', 'assay', 'disease', 'sex', 'tissue', 'self_reported_ethnicity', 'development_stage', 'observation_joinid'
    var: 'feature_is_filtered', 'feature_name', 'feature_reference', 'feature_biotype', 'feature_length', 'feature_type', 'Ensemble_ID'
    uns: 'citation', 'default_embedding', 'organism', 'organism_ontology_term_id', 'schema_reference', 'schema_version', 'title'
    obsm: 'X_umap'
    layers: 'raw'

adata.obs.head().T

import piaso

stderr

.../site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

sc.pl.umap(adata,
           color=['cell_type'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

sc.pl.umap(adata,
           color=['Annotation_Level4'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

sc.pl.umap(adata,
           color=['Annotation_Level3'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

%%time
piaso.tl.infog(adata,
               layer='raw',
               n_top_genes=3000,)

The normalized data is saved as `infog` in `adata.layers`.
The highly variable genes are saved as `highly_variable` in `adata.var`.
Finished INFOG normalization.
CPU times: user 3min 37s, sys: 2min 21s, total: 5min 58s
Wall time: 5min 59s

piaso.pp.table(adata.obs['cell_type'])

{'T cell': 54800,
 'myeloid cell': 26020,
 'natural killer cell': 23234,
 'B cell': 9144,
 'platelet': 14799,
 'plasma cell': 1672,
 'naive B cell': 36778,
 'memory B cell': 35114,
 'mature B cell': 4850,
 'hematopoietic stem cell': 135,
 'CD14-positive monocyte': 202124,
 'CD14-low, CD16-positive monocyte': 46628,
 'CD1c-positive myeloid dendritic cell': 13207,
 'CD141-positive myeloid dendritic cell': 554,
 'pre-conventional dendritic cell': 894,
 'plasmacytoid dendritic cell': 3796,
 'CD16-positive, CD56-dim natural killer cell, human': 151655,
 'CD8-positive, alpha-beta cytotoxic T cell': 46528,
 'CD8-positive, alpha-beta T cell': 53658,
 'CD16-negative, CD56-bright natural killer cell, human': 6152,
 'CD4-positive, alpha-beta cytotoxic T cell': 13594,
 'gamma-delta T cell': 30695,
 'CD8-positive, alpha-beta memory T cell': 28138,
 'CD4-positive, alpha-beta T cell': 40100,
 'innate lymphoid cell': 707,
... [7 more lines]

piaso.pp.table(adata.obs['tissue'])

{'blood': 1265624}

piaso.pp.table(adata.obs['disease'])

{'normal': 1265624}

piaso.pp.table(adata.obs['assay'])

{"10x 5' v2": 1265624}

piaso.pp.table(adata.obs['library_id'])

{'c18f20cd-6317-4059-bc5a-5341fe134124': 8341,
 '8c929df1-d96a-437e-94cf-795ba97ba226': 11777,
 'a539496c-999e-4935-865e-2c1c7506bbc9': 13902,
 '91f42a59-eeb0-4577-a6d0-52f17b4ea3b3': 12056,
 '25e6ef7a-298c-42d5-9be8-b09932e1fd9d': 12235,
 '10a42edf-8fd3-4e25-900c-5a8e4890bcc8': 16351,
 '3caa9824-a0cb-4d3c-a642-b2f374f690b5': 12486,
 'a6c4ebc3-beac-4ebe-8e27-85afbe0d55b2': 18336,
 '5205817f-44c5-468b-9d41-043ecacb7dbb': 17684,
 '163573e7-c093-4a02-8ce4-b38e61d4d78e': 15374,
 '1833343d-c74b-409a-a2ad-15bdfaddf876': 16129,
 'db0a8252-6f37-47d4-98d6-806b181c1520': 17231,
 '745201e8-508b-4484-b179-6a3dc332895a': 16823,
 '8eb9096b-f4be-4fbf-91a3-402621f6d7d2': 17066,
 '688adb38-2ca3-4efa-8183-a1130f6c7801': 18609,
 '354ba71c-599c-4cd0-8f31-07d7b5676ded': 16138,
 'fef1f5a0-b108-435e-a754-3cc995daaa1f': 15994,
 '7b848d92-f5d1-4b53-a681-889c581d5ec9': 17178,
 'd43d2448-bf59-4267-a458-1ceb83096c67': 17712,
 'bb48421f-5cb1-4f12-b725-dcfa41bc700b': 10244,
 '7da3c8a8-dfcf-4d98-bcdb-63281d8903f9': 12889,
 '1d7edd49-e8ab-494d-8d33-f452f138e945': 16113,
 '0ebbd2ac-4260-4342-af3b-ff24012e8681': 17663,
 '091b1af8-3dc5-494f-a485-4b647c9f2bfa': 17975,
 '57aa8942-f88d-4b83-9888-f6f5917a84a9': 17443,
... [68 more lines]

adata.X.data

array([1.21232039, 1.21232039, 1.21232039, ..., 2.99794589, 1.82356588,
       3.96914141], shape=(2544339517,))

%%time
piaso.tl.runGDRParallel(adata,
                        batch_key='library_id',
                        groupby=None,
                        n_gene=20,
                        mu=10,
                        resolution=3.0,
                        layer='infog',
                        infog_layer='raw',
                        score_layer='infog',
                        scoring_method='piaso',
                        use_highly_variable=True,
                        n_highly_variable_genes=5000,
                        n_svd_dims=50,
                        key_added='X_gdr',
                        max_workers=32,
                        calculate_score_multiBatch = False,
                        verbosity=0)

stderr

Calculating marker genes: 100%|██████████| 93/93 [11:38<00:00,  7.51s/batch]
Calculating cell embeddings: 100%|██████████| 93/93 [21:13:01<00:00, 821.31s/batch]

The cell embeddings calculated by GDR were saved as `X_gdr` in adata.obsm.
CPU times: user 14min 15s, sys: 5min 11s, total: 19min 27s
Wall time: 21h 26min 6s

adata

AnnData object with n_obs × n_vars = 1265624 × 35477
    obs: 'reference_genome', 'gene_annotation_version', 'alignment_software', 'intronic_reads_counted', 'library_id', 'assay_ontology_term_id', 'sequenced_fragment', 'cell_number_loaded', 'institute', 'is_primary_data', 'cell_type_ontology_term_id', 'author_cell_type', 'sample_id', 'sample_preservation_method', 'tissue_ontology_term_id', 'development_stage_ontology_term_id', 'sample_collection_method', 'donor_BMI_at_collection', 'tissue_type', 'suspension_derivation_process', 'suspension_enriched_cell_types', 'cell_viability_percentage', 'suspension_uuid', 'suspension_type', 'donor_id', 'self_reported_ethnicity_ontology_term_id', 'donor_living_at_sample_collection', 'disease_ontology_term_id', 'sex_ontology_term_id', 'nCount_RNA', 'nFeature_RNA', 'pMito', 'NODG', 'nUMI', 'Country', 'Annotation_Level1', 'Annotation_Level2', 'Annotation_Level3', 'Annotation_Level4', 'Smoking Status', 'cell_type', 'assay', 'disease', 'sex', 'tissue', 'self_reported_ethnicity', 'development_stage', 'observation_joinid'
    var: 'feature_is_filtered', 'feature_name', 'feature_reference', 'feature_biotype', 'feature_length', 'feature_type', 'Ensemble_ID', 'infog_var', 'highly_variable'
    uns: 'Annotation_Level3_colors', 'Annotation_Level4_colors', 'cell_type_colors', 'citation', 'default_embedding', 'organism', 'organism_ontology_term_id', 'schema_reference', 'schema_version', 'title', 'gdr', 'neighbors', 'umap', 'assay_colors'
    obsm: 'X_umap', 'X_gdr', 'X_umap_author'
    layers: 'raw', 'infog'
    obsp: 'distances', 'connectivities'

adata.obsm['X_umap_author']=adata.obsm['X_umap'].copy()

%%time
sc.pp.neighbors(adata,
                use_rep='X_gdr',
                n_neighbors=15,
                random_state=10,
                knn=True,
                method="umap")
sc.tl.umap(adata)

computing neighbors
    finished: added
    'X_umap', UMAP coordinates (adata.obsm)
    'umap', UMAP parameters (adata.uns) (0:35:54)
CPU times: user 2h 22min, sys: 25min 7s, total: 2h 47min 8s
Wall time: 2h 31min 48s

sc.pl.umap(adata,
           color=['assay'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

sc.pl.umap(adata,
           color=['cell_type'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

sc.pl.embedding(adata,
           basis='X_umap_author',
           color=['cell_type'],
           palette=piaso.pl.color.d_color4,
           cmap=piaso.pl.color.c_color4,
           # size=10,
           ncols=1,
           frameon=False)

adata.obsm['X_umap_gdr']=adata.obsm['X_umap'].copy()

adata

AnnData object with n_obs × n_vars = 1265624 × 35477
    obs: 'reference_genome', 'gene_annotation_version', 'alignment_software', 'intronic_reads_counted', 'library_id', 'assay_ontology_term_id', 'sequenced_fragment', 'cell_number_loaded', 'institute', 'is_primary_data', 'cell_type_ontology_term_id', 'author_cell_type', 'sample_id', 'sample_preservation_method', 'tissue_ontology_term_id', 'development_stage_ontology_term_id', 'sample_collection_method', 'donor_BMI_at_collection', 'tissue_type', 'suspension_derivation_process', 'suspension_enriched_cell_types', 'cell_viability_percentage', 'suspension_uuid', 'suspension_type', 'donor_id', 'self_reported_ethnicity_ontology_term_id', 'donor_living_at_sample_collection', 'disease_ontology_term_id', 'sex_ontology_term_id', 'nCount_RNA', 'nFeature_RNA', 'pMito', 'NODG', 'nUMI', 'Country', 'Annotation_Level1', 'Annotation_Level2', 'Annotation_Level3', 'Annotation_Level4', 'Smoking Status', 'cell_type', 'assay', 'disease', 'sex', 'tissue', 'self_reported_ethnicity', 'development_stage', 'observation_joinid'
    var: 'feature_is_filtered', 'feature_name', 'feature_reference', 'feature_biotype', 'feature_length', 'feature_type', 'Ensemble_ID', 'infog_var', 'highly_variable'
    uns: 'Annotation_Level3_colors', 'Annotation_Level4_colors', 'cell_type_colors', 'citation', 'default_embedding', 'organism', 'organism_ontology_term_id', 'schema_reference', 'schema_version', 'title', 'gdr', 'neighbors', 'umap', 'assay_colors'
    obsm: 'X_umap', 'X_gdr', 'X_umap_author', 'X_umap_gdr'
    layers: 'raw', 'infog'
    obsp: 'distances', 'connectivities'

adata.write(save_dir+'/'+prefix+'_gdr.h5ad')

save_dir+'/'+prefix+'_gdr.h5ad'

'.../Result/single-cell/Methods/DataProcessing/AsianImmuneDiversityAtlasPhase1v2/AsianImmuneDiversityAtlasPhase1v2_gdr.h5ad'