From 9d60f502cbada08f241e70196aeac4a4cdedd092 Mon Sep 17 00:00:00 2001
From: Amit Kesarwani <93291915+kesarwam@users.noreply.github.com>
Date: Fri, 10 May 2024 14:21:29 -0700
Subject: [PATCH] Updated Data Lineage demo to use Python Wrapper (#190)
Co-authored-by: Iddo Avneri
The test that fails (R notebook) is already removed in the futrue version.
---
00_notebooks/data-lineage.ipynb | 891 +++++++++-----------------------
1 file changed, 235 insertions(+), 656 deletions(-)
diff --git a/00_notebooks/data-lineage.ipynb b/00_notebooks/data-lineage.ipynb
index ad24d23c2..ec93f8405 100644
--- a/00_notebooks/data-lineage.ipynb
+++ b/00_notebooks/data-lineage.ipynb
@@ -46,7 +46,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"id": "79a78d9a-10b0-4ffa-b501-213876abe1ad",
"metadata": {
"tags": [],
@@ -73,7 +73,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "f6a5b298-1879-4441-a607-0e64289bf90e",
"metadata": {
"tags": [],
@@ -106,9 +106,17 @@
"**(you shouldn't need to change anything in this section, just run it)**"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "fb3f61b0-770c-43f3-a227-787354a1fd2f",
+ "metadata": {},
+ "source": [
+ "### lakeFS repository name"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"id": "a45fbab3-5f98-46ad-8e12-9021ed2bf81a",
"metadata": {
"tags": [],
@@ -121,6 +129,53 @@
"repo_name = \"data-lineage\""
]
},
+ {
+ "cell_type": "markdown",
+ "id": "c093db8e-68d9-409f-bde7-73ee4ceca5a5",
+ "metadata": {},
+ "source": [
+ "### Versioning Information"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2d78c797-911d-42f2-be06-f898f4d31c58",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "productionBranch = \"main\"\n",
+ "ingestionBranch1 = \"ingest1\"\n",
+ "ingestionBranch2 = \"ingest2\"\n",
+ "transformationBranch = \"transformation\"\n",
+ "newPath = \"partitioned_data\"\n",
+ "fileName = \"Employees.csv\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d00b4483-5365-452d-a122-d08c54b0e433",
+ "metadata": {},
+ "source": [
+ "### Import libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "60cd210f-b9ce-4052-a0cd-b679d8cf43af",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import lakefs\n",
+ "from assets.lakefs_demo import print_commit"
+ ]
+ },
{
"cell_type": "markdown",
"id": "70fcf734-8b50-4c82-a42d-a16c3a007a38",
@@ -128,12 +183,12 @@
"tags": []
},
"source": [
- "### Create lakeFSClient"
+ "### Set environment variables"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"id": "4795b1e7-761e-4f7a-840b-982f99ff3c6a",
"metadata": {
"tags": [],
@@ -143,17 +198,9 @@
},
"outputs": [],
"source": [
- "import lakefs_client\n",
- "from lakefs_client.models import *\n",
- "from lakefs_client.client import LakeFSClient\n",
- "\n",
- "# lakeFS credentials and endpoint\n",
- "configuration = lakefs_client.Configuration()\n",
- "configuration.username = lakefsAccessKey\n",
- "configuration.password = lakefsSecretKey\n",
- "configuration.host = lakefsEndPoint\n",
- "\n",
- "lakefs = LakeFSClient(configuration)"
+ "os.environ[\"LAKECTL_SERVER_ENDPOINT_URL\"] = lakefsEndPoint\n",
+ "os.environ[\"LAKECTL_CREDENTIALS_ACCESS_KEY_ID\"] = lakefsAccessKey\n",
+ "os.environ[\"LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY\"] = lakefsSecretKey"
]
},
{
@@ -161,12 +208,12 @@
"id": "41c991ae-c9d1-4ca8-934e-ea3d80285037",
"metadata": {},
"source": [
- "#### Verify lakeFS credentials by getting lakeFS version"
+ "### Verify lakeFS credentials by getting lakeFS version"
]
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"id": "e3bbe675-3c5e-41c7-8471-7a8a90253827",
"metadata": {
"tags": [],
@@ -174,26 +221,15 @@
"languageId": "python"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Verifying lakeFS credentials…\n",
- "…✅lakeFS credentials verified\n",
- "\n",
- "ℹ️lakeFS version 0.104.0\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"print(\"Verifying lakeFS credentials…\")\n",
"try:\n",
- " v=lakefs.config.get_config()\n",
+ " v=lakefs.client.Client().version\n",
"except:\n",
" print(\"🛑 failed to get lakeFS version\")\n",
"else:\n",
- " print(f\"…✅lakeFS credentials verified\\n\\nℹ️lakeFS version {v['version_config']['version']}\")"
+ " print(f\"…✅lakeFS credentials verified\\n\\nℹ️lakeFS version {v}\")"
]
},
{
@@ -206,42 +242,15 @@
},
{
"cell_type": "code",
- "execution_count": 6,
- "id": "384aa4d3-038a-4a56-b30e-c536e3911478",
+ "execution_count": null,
+ "id": "ebe86116-095c-4058-9e1e-aca78680d61d",
"metadata": {
- "tags": [],
- "vscode": {
- "languageId": "python"
- }
+ "tags": []
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Repository data-lineage does not exist, so going to try and create it now.\n",
- "Created new repo data-lineage using storage namespace s3://example/data-lineage\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "from lakefs_client.exceptions import NotFoundException\n",
- "\n",
- "try:\n",
- " repo=lakefs.repositories.get_repository(repo_name)\n",
- " print(f\"Found existing repo {repo.id} using storage namespace {repo.storage_namespace}\")\n",
- "except NotFoundException as f:\n",
- " print(f\"Repository {repo_name} does not exist, so going to try and create it now.\")\n",
- " try:\n",
- " repo=lakefs.repositories.create_repository(repository_creation=RepositoryCreation(name=repo_name,\n",
- " storage_namespace=f\"{storageNamespace}/{repo_name}\"))\n",
- " print(f\"Created new repo {repo.id} using storage namespace {repo.storage_namespace}\")\n",
- " except lakefs_client.ApiException as e:\n",
- " print(f\"Error creating repo {repo_name}. Error is {e}\")\n",
- " os._exit(00)\n",
- "except lakefs_client.ApiException as e:\n",
- " print(f\"Error getting repo {repo_name}: {e}\")\n",
- " os._exit(00)"
+ "repo = lakefs.Repository(repo_name).create(storage_namespace=f\"{storageNamespace}/{repo_name}\", default_branch=productionBranch, exist_ok=True)\n",
+ "print(repo)"
]
},
{
@@ -254,7 +263,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"id": "6e7196c9-370a-441a-8ea3-02ee03484991",
"metadata": {
"tags": [],
@@ -262,41 +271,7 @@
"languageId": "python"
}
},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- "
SparkSession - in-memory
\n",
- " \n",
- "
\n",
- "
SparkContext
\n",
- "\n",
- "
Spark UI
\n",
- "\n",
- "
\n",
- " - Version
\n",
- " v3.3.2- Master
\n",
- " local[*]- AppName
\n",
- " lakeFS / Jupyter
\n",
- "
\n",
- " \n",
- "