diff --git a/tensorflow/notebooks/Filesystem/HopsFSOperations.ipynb b/tensorflow/notebooks/Filesystem/HopsFSOperations.ipynb index f693ab0d..2aae2716 100644 --- a/tensorflow/notebooks/Filesystem/HopsFSOperations.ipynb +++ b/tensorflow/notebooks/Filesystem/HopsFSOperations.ipynb @@ -25,37 +25,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting Spark application\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
10application_1537374274509_0011pysparkidleLinkLink✔
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SparkSession available as 'spark'.\n" - ] - } - ], + "outputs": [], "source": [ "from hops import hdfs" ] @@ -71,19 +43,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "project user: HopsFS_Operations__meb10000\n", - "project name: HopsFS_Operations\n", - "project path: hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/" - ] - } - ], + "outputs": [], "source": [ "project_user = hdfs.project_user()\n", "project_name = hdfs.project_name()\n", @@ -100,24 +62,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "logs README: *This is an auto-generated README.md file for your Dataset!*\n", - "To replace it, go into your DataSet and edit the README.md file.\n", - "\n", - "*Resources* DataSet\n", - "===\n", - "\n", - "## Contains resources used by jobs, for example, jar files.\n", - "logs README dumped: test" - ] - } - ], + "outputs": [], "source": [ "logs_README = hdfs.load(\"Logs/README.md\")\n", "print(\"logs README: {}\".format(logs_README.decode(\"utf-8\")))\n", @@ -135,18 +82,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "copied file from local to hdfs: test\n", - "copied file from hdfs to local: test" - ] - } - ], + "outputs": [], "source": [ "# creates file in current working directory with a string\n", "with open('test.txt', 'w') as f:\n", @@ -169,19 +107,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt']\n", - "[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md']\n", - "[{'size': 211L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', 'replication': 3L, 'last_mod': 1537437288L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', 'last_access': 1537441705L, 'block_size': 134217728L, 'permissions': 777L}, {'size': 227L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', 'replication': 3L, 'last_mod': 1537433132L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', 'last_access': 1537437246L, 'block_size': 134217728L, 'permissions': 1016L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', 'replication': 3L, 'last_mod': 1537445071L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', 'last_access': 1537445071L, 'block_size': 134217728L, 'permissions': 420L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', 'replication': 3L, 'last_mod': 1537443005L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', 'last_access': 1537443005L, 'block_size': 134217728L, 'permissions': 420L}, {'size': 4L, 'kind': u'file', 'group': u'HopsFS_Operations__Logs', 'name': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', 'replication': 3L, 'last_mod': 1537444773L, 'owner': u'HopsFS_Operations__meb10000', 'path': u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', 'last_access': 1537444773L, 'block_size': 134217728L, 'permissions': 420L}]" - ] - } - ], + "outputs": [], "source": [ "logs_files = hdfs.ls(\"Logs/\")\n", "print(logs_files)\n", @@ -200,17 +128,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']" - ] - } - ], + "outputs": [], "source": [ "hdfs.cp(\"Resources/test.txt\", \"Logs/\")\n", "logs_files = hdfs.ls(\"Logs/\")\n", @@ -226,18 +146,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "files prior to delete: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test_dir']\n", - "files after to delete: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']" - ] - } - ], + "outputs": [], "source": [ "hdfs.mkdir(\"Logs/test_dir\")\n", "logs_files_prior_delete = hdfs.ls(\"Logs/\")\n", @@ -256,20 +167,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "files prior to move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n", - "files after move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n", - "files prior to rename: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']\n", - "files after move: [u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README2.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README_dump_test.md', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test4.txt', u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/test5.txt']" - ] - } - ], + "outputs": [], "source": [ "logs_files_prior_move = hdfs.ls(\"Logs/\")\n", "print(\"files prior to move: {}\".format(logs_files_prior_move))\n", @@ -292,18 +192,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "file permissions prior to chmod: 1100001001\n", - "file permissions after to chmod: 1010111100" - ] - } - ], + "outputs": [], "source": [ "import stat\n", "file_stat = hdfs.stat(\"Logs/README.md\")\n", @@ -326,18 +217,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "file_stat: StatResult(st_atime=1537441705, st_blksize=134217728, st_blocks=1L, st_ctime=0, st_dev=0L, st_gid='HopsFS_Operations__Logs', st_ino=0, st_mode=777, st_mtime=1537437288, st_nlink=1, st_size=211L, st_uid='HopsFS_Operations__meb10000', kind='file', name=u'hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/README.md', replication=3)\n", - "file access: False" - ] - } - ], + "outputs": [], "source": [ "file_stat = hdfs.stat(\"Logs/README.md\")\n", "print(\"file_stat: {}\".format(file_stat))\n", @@ -354,28 +236,38 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hdfs://10.0.2.15:8020/Projects/HopsFS_Operations/Logs/" - ] - } - ], + "outputs": [], "source": [ "abs_path = hdfs.abs_path(\"Logs/\")\n", "print(abs_path)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check if file/folder exists" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "hdfs.exists(\"Logs/\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hdfs.exists(\"Not_Existing/neither_am_i\")" + ] } ], "metadata": {