diff --git a/07_querying_lakehouse.ipynb b/07_querying_lakehouse.ipynb index 02dc823..92bfc57 100644 --- a/07_querying_lakehouse.ipynb +++ b/07_querying_lakehouse.ipynb @@ -27,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -386,30 +386,30 @@ "name": "stdout", "output_type": "stream", "text": [ - "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+----------------------------+\n", - "| SequenceRunName|SubjectID|Gender|Phenotype|StudyID| DiseaseCode| SNOMED|SampleID|CHROM| REF| ALT|array_size(alternateAlleles)|\n", - "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+----------------------------+\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| AT| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| A| [G]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [C]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [G]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [G]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [G]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [C]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [C]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| A| [G]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G|[GAA]| 1|\n", - "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9|AATGTGGGGCATACACAT| [A]| 1|\n", - "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+----------------------------+\n", + "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+-------+\n", + "| SequenceRunName|SubjectID|Gender|Phenotype|StudyID| DiseaseCode| SNOMED|SampleID|CHROM| REF| ALT|ALT_cnt|\n", + "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+-------+\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| AT| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| A| [G]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [C]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [G]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [G]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [G]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [C]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| T| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [C]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| C| [T]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| A| [G]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G| [A]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9| G|[GAA]| 1|\n", + "|221007_A00130_000...| SBJ00001|Female| normal|NA12878|MONDO:0007254|429740004| NA12878| chr9|AATGTGGGGCATACACAT| [A]| 1|\n", + "+--------------------+---------+------+---------+-------+-------------+---------+--------+-----+------------------+-----+-------+\n", "only showing top 20 rows\n", "\n" ] @@ -418,7 +418,7 @@ "source": [ "spark.sql(\"select \\\n", " m.SequenceRunName, m.SubjectID, m.Gender, m.Phenotype, m.StudyID, m.DiseaseCode, m.SNOMED, m.SampleID, \\\n", - " s.contigName as CHROM, s.referenceAllele as REF, s.alternateAlleles as ALT, array_size(s.alternateAlleles) \\\n", + " s.contigName as CHROM, s.referenceAllele as REF, s.alternateAlleles as ALT, array_size(s.alternateAlleles) as ALT_cnt \\\n", "from metadata_table as m \\\n", "join somatic_table as s on s.genotypes_sampleId = m.SampleID\").show()" ]