Skip to content
This repository has been archived by the owner on Apr 14, 2023. It is now read-only.

Commit

Permalink
Add exists example to HopsFS
Browse files Browse the repository at this point in the history
  • Loading branch information
@robzor92
robzor92 committed Oct 19, 2018
1 parent 9454fd8 commit 9079aff
Showing 1 changed file with 41 additions and 149 deletions.
190 changes: 41 additions & 149 deletions tensorflow/notebooks/Filesystem/HopsFSOperations.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,37 +25,9 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Starting Spark application\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
"<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>10</td><td>application_1537374274509_0011</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://hopsworks0:8088/proxy/application_1537374274509_0011/\">Link</a></td><td><a target=\"_blank\" href=\"http://hopsworks0:8042/node/containerlogs/container_e01_1537374274509_0011_01_000001/HopsFS_Operations__meb10000\">Link</a></td><td>✔</td></tr></table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"SparkSession available as 'spark'.\n"
]
}
],
"outputs": [],
"source": [
"from hops import hdfs"
]
Expand All @@ -71,19 +43,9 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"project user: HopsFS_Operations__meb10000\n",
"project name: HopsFS_Operations\n",
"project path: hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/"
]
}
],
"outputs": [],
"source": [
"project_user = hdfs.project_user()\n",
"project_name = hdfs.project_name()\n",
Expand All @@ -100,24 +62,9 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"logs README: *This is an auto-generated README.md file for your Dataset!*\n",
"To replace it, go into your DataSet and edit the README.md file.\n",
"\n",
"*Resources* DataSet\n",
"===\n",
"\n",
"## Contains resources used by jobs, for example, jar files.\n",
"logs README dumped: test"
]
}
],
"outputs": [],
"source": [
"logs_README = hdfs.load(\"Logs/README.md\")\n",
"print(\"logs README: {}\".format(logs_README.decode(\"utf-8\")))\n",
Expand All @@ -135,18 +82,9 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"copied file from local to hdfs: test\n",
"copied file from hdfs to local: test"
]
}
],
"outputs": [],
"source": [
"# creates file in current working directory with a string\n",
"with open('test.txt', 'w') as f:\n",
Expand All @@ -169,19 +107,9 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt']\n",
"[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md']\n",
"[{'size': 211L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', 'replication': 3L, 'last_mod': 1537437288L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', 'last_access': 1537441705L, 'block_size': 134217728L, 'permissions': 777L}, {'size': 227L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', 'replication': 3L, 'last_mod': 1537433132L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', 'last_access': 1537437246L, 'block_size': 134217728L, 'permissions': 1016L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', 'replication': 3L, 'last_mod': 1537445071L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', 'last_access': 1537445071L, 'block_size': 134217728L, 'permissions': 420L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', 'replication': 3L, 'last_mod': 1537443005L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', 'last_access': 1537443005L, 'block_size': 134217728L, 'permissions': 420L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', 'replication': 3L, 'last_mod': 1537444773L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', 'last_access': 1537444773L, 'block_size': 134217728L, 'permissions': 420L}]"
]
}
],
"outputs": [],
"source": [
"logs_files = hdfs.ls(\"Logs/\")\n",
"print(logs_files)\n",
Expand All @@ -200,17 +128,9 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']"
]
}
],
"outputs": [],
"source": [
"hdfs.cp(\"Resources/test.txt\", \"Logs/\")\n",
"logs_files = hdfs.ls(\"Logs/\")\n",
Expand All @@ -226,18 +146,9 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"files prior to delete: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test_dir']\n",
"files after to delete: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']"
]
}
],
"outputs": [],
"source": [
"hdfs.mkdir(\"Logs/test_dir\")\n",
"logs_files_prior_delete = hdfs.ls(\"Logs/\")\n",
Expand All @@ -256,20 +167,9 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"files prior to move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n",
"files after move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n",
"files prior to rename: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n",
"files after move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']"
]
}
],
"outputs": [],
"source": [
"logs_files_prior_move = hdfs.ls(\"Logs/\")\n",
"print(\"files prior to move: {}\".format(logs_files_prior_move))\n",
Expand All @@ -292,18 +192,9 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"file permissions prior to chmod: 1100001001\n",
"file permissions after to chmod: 1010111100"
]
}
],
"outputs": [],
"source": [
"import stat\n",
"file_stat = hdfs.stat(\"Logs/README.md\")\n",
Expand All @@ -326,18 +217,9 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"file_stat: StatResult(st_atime=1537441705, st_blksize=134217728, st_blocks=1L, st_ctime=0, st_dev=0L, st_gid='HopsFS_Operations__Logs', st_ino=0, st_mode=777, st_mtime=1537437288, st_nlink=1, st_size=211L, st_uid='HopsFS_Operations__meb10000', kind='file', name=u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', replication=3)\n",
"file access: False"
]
}
],
"outputs": [],
"source": [
"file_stat = hdfs.stat(\"Logs/README.md\")\n",
"print(\"file_stat: {}\".format(file_stat))\n",
Expand All @@ -354,28 +236,38 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/"
]
}
],
"outputs": [],
"source": [
"abs_path = hdfs.abs_path(\"Logs/\")\n",
"print(abs_path)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Check if file/folder exists"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"hdfs.exists(\"Logs/\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"hdfs.exists(\"Not_Existing/neither_am_i\")"
]
}
],
"metadata": {
Expand Down

0 comments on commit 9079aff

Please sign in to comment.