Skip to content

Commit

Permalink
Linting
Browse files Browse the repository at this point in the history
  • Loading branch information
Omar Marzouk committed Jun 5, 2022
1 parent 8e3a32f commit d02b996
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 30 deletions.
130 changes: 122 additions & 8 deletions docs/tutorials/daos.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,31 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### Copyright 2022 The TensorFlow IO Authors."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
"# you may not use this file except in compliance with the License.\n",
"# You may obtain a copy of the License at\n",
"#\n",
"# https://www.apache.org/licenses/LICENSE-2.0\n",
"#\n",
"# Unless required by applicable law or agreed to in writing, software\n",
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
"# See the License for the specific language governing permissions and\n",
"# limitations under the License."
]
},
{
"cell_type": "markdown",
"id": "1f9e30da",
Expand All @@ -10,6 +36,26 @@
"# DAOS Filesystem with Tensorflow (Using MNIST)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://www.tensorflow.org/io/tutorials/daos\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/io/blob/master/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://github.com/tensorflow/io/blob/master/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
" </td>\n",
" <td>\n",
" <a href=\"https://storage.googleapis.com/tensorflow_docs/io/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
" </td>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "22b37505",
Expand All @@ -21,7 +67,7 @@
"\n",
"This tutorial shows how to use read and write files on [DAOS Filesystem](https://docs.daos.io/) with TensorFlow, through TensorFlow IO's DAOS file system integration.\n",
"\n",
"A machine running DAOS natively or through a [docker emulator](https://github.com/daos-stack/daos/tree/master/utils/docker) is needed to run this tutorial and/or use the Tensorflow IO DAOS integration. The DAOS Pool and Container used for this tutorial will be created and deleted within this tutorial, where we will be training and testing a simple Neural Network on the MNIST Dataset loaded from the DAOS File System Plugin.\n",
"A machine running DAOS natively or through a [docker emulator](https://github.com/daos-stack/daos/tree/master/utils/docker) is needed to run this tutorial and/or use the Tensorflow IO DAOS integration. The DAOS Pool and Container used for this tutorial will be created and deleted within this tutorial, where you will be training and testing a simple Neural Network on the MNIST Dataset loaded from the DAOS File System Plugin.\n",
"\n",
"The pool and container id or label are part of the filename uri:\n",
"```\n",
Expand Down Expand Up @@ -52,12 +98,59 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "5e35916b",
"metadata": {
"id": "5de1951509cb"
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: tensorflow-io in /home/omar/.local/lib/python3.8/site-packages (0.20.0)\n",
"Requirement already satisfied: tensorflow<2.7.0,>=2.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow-io) (2.6.0)\n",
"Requirement already satisfied: tensorflow-io-gcs-filesystem==0.20.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow-io) (0.20.0)\n",
"Requirement already satisfied: gast==0.4.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.0)\n",
"Requirement already satisfied: grpcio<2.0,>=1.37.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.39.0)\n",
"Requirement already satisfied: protobuf>=3.9.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.17.3)\n",
"Requirement already satisfied: tensorboard~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
"Requirement already satisfied: tensorflow-estimator~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
"Requirement already satisfied: typing-extensions~=3.7.4 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.7.4.3)\n",
"Requirement already satisfied: termcolor~=1.1.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.1.0)\n",
"Requirement already satisfied: wrapt~=1.12.1 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.12.1)\n",
"Requirement already satisfied: google-pasta~=0.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.2.0)\n",
"Requirement already satisfied: keras~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
"Requirement already satisfied: six~=1.15.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.15.0)\n",
"Requirement already satisfied: numpy~=1.19.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.19.5)\n",
"Requirement already satisfied: opt-einsum~=3.3.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.3.0)\n",
"Requirement already satisfied: wheel~=0.35 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.37.0)\n",
"Requirement already satisfied: astunparse~=1.6.3 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.6.3)\n",
"Requirement already satisfied: absl-py~=0.10 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.13.0)\n",
"Requirement already satisfied: keras-preprocessing~=1.1.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.1.2)\n",
"Requirement already satisfied: h5py~=3.1.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.1.0)\n",
"Requirement already satisfied: clang~=5.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (5.0)\n",
"Requirement already satisfied: flatbuffers~=1.12.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.12)\n",
"Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.6)\n",
"Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.8.0)\n",
"Requirement already satisfied: markdown>=2.6.8 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.3.4)\n",
"Requirement already satisfied: google-auth<2,>=1.6.3 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.35.0)\n",
"Requirement already satisfied: setuptools>=41.0.0 in /usr/lib/python3/dist-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (45.2.0)\n",
"Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.6.1)\n",
"Requirement already satisfied: requests<3,>=2.21.0 in /usr/lib/python3/dist-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.22.0)\n",
"Requirement already satisfied: werkzeug>=0.11.15 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.0.1)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.2.8)\n",
"Requirement already satisfied: rsa<5,>=3.1.4 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (4.7.2)\n",
"Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (4.2.2)\n",
"Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/omar/.local/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.3.0)\n",
"Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/omar/.local/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.8)\n",
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.1.0)\n",
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n"
]
}
],
"source": [
"try:\n",
" %tensorflow_version 2.x \n",
Expand All @@ -79,12 +172,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "79528fed",
"metadata": {
"id": "fb83b02da201"
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/usr/bin/sh: 1: dmg: not found\n",
"/usr/bin/sh: 1: daos: not found\n"
]
}
],
"source": [
"!dmg -i pool create -s 500M TEST_POOL\n",
"!daos cont create --pool=TEST_POOL --type=POSIX TEST_CONT"
Expand Down Expand Up @@ -128,7 +230,7 @@
},
"outputs": [],
"source": [
"dfs_url = \"dfs://TEST_POOL/TEST_CONT/\" # This the path we'll be using to load and access the dataset\n",
"dfs_url = \"dfs://TEST_POOL/TEST_CONT/\" # This the path you'll be using to load and access the dataset\n",
"pwd = !pwd\n",
"posix_url = pwd[0] + \"/tests/test_dfs/\""
]
Expand Down Expand Up @@ -162,7 +264,7 @@
"id": "b40e9a30808c"
},
"source": [
"Copying the Data from the POSIX Filesystem to the DAOS Filesystem under the pool and container we just created"
"Copying the Data from the POSIX Filesystem to the DAOS Filesystem under the pool and container you just created"
]
},
{
Expand Down Expand Up @@ -270,7 +372,7 @@
"id": "4129e8e2c1b4"
},
"source": [
"Compiling the model we just built"
"Compiling the model you just built"
]
},
{
Expand Down Expand Up @@ -467,6 +569,18 @@
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
Expand Down
31 changes: 14 additions & 17 deletions tensorflow_io/core/filesystems/dfs/dfs_filesystem.cc
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
#include "tensorflow_io/core/filesystems/dfs/dfs_utils.h"

#include <stdio.h>

#include "tensorflow_io/core/filesystems/dfs/dfs_utils.h"
#undef NDEBUG
#include <cassert>

namespace tensorflow {
namespace io {
namespace dfs {


// SECTION 1. Implementation for `TF_RandomAccessFile`
// ----------------------------------------------------------------------------
namespace tf_random_access_file {
typedef struct DFSRandomAccessFile {
std::string dfs_path;
dfs_t* daos_fs;
dfs_obj_t *daos_file;
dfs_obj_t* daos_file;
std::vector<ReadBuffer> buffers;
daos_size_t file_size;
daos_handle_t mEventQueueHandle{};
Expand Down Expand Up @@ -77,7 +76,7 @@ int64_t Read(const TF_RandomAccessFile* file, uint64_t offset, size_t n,
if (read_buf.CacheHit(curr_offset)) {
read_bytes = read_buf.CopyFromCache(ret, ret_offset, curr_offset, n,
dfs_file->file_size, status);
break;
break;
}
}

Expand All @@ -96,8 +95,8 @@ int64_t Read(const TF_RandomAccessFile* file, uint64_t offset, size_t n,
size_t async_offset = curr_offset;
for (size_t i = 0; i < dfs_file->buffers.size(); i++) {
if (async_offset > dfs_file->file_size) break;
dfs_file->buffers[i].ReadAsync(dfs_file->daos_fs,
dfs_file->daos_file, async_offset);
dfs_file->buffers[i].ReadAsync(dfs_file->daos_fs, dfs_file->daos_file,
async_offset);
async_offset += BUFF_SIZE;
}
}
Expand All @@ -113,18 +112,18 @@ namespace tf_writable_file {
typedef struct DFSWritableFile {
std::string dfs_path;
dfs_t* daos_fs;
dfs_obj_t *daos_file;
dfs_obj_t* daos_file;
daos_size_t file_size;
bool size_known;

DFSWritableFile(std::string dfs_path, dfs_t* file_system, dfs_obj_t* obj)
: dfs_path(std::move(dfs_path)) {
daos_fs = file_system;
daos_file = obj;
size_known=false;
size_known = false;
}

int get_file_size(daos_size_t &size) {
int get_file_size(daos_size_t& size) {
if (!size_known) {
int rc = dfs_get_size(daos_fs, daos_file, &file_size);
if (rc != 0) {
Expand All @@ -141,9 +140,7 @@ typedef struct DFSWritableFile {
size_known = true;
}

void unset_file_size(void) {
size_known = false;
}
void unset_file_size(void) { size_known = false; }
} DFSWritableFile;

void Cleanup(TF_WritableFile* file) {
Expand Down Expand Up @@ -171,8 +168,8 @@ void Append(const TF_WritableFile* file, const char* buffer, size_t n,
return;
}

rc = dfs_write(dfs_file->daos_fs, dfs_file->daos_file, &wsgl,
cur_file_size, NULL);
rc = dfs_write(dfs_file->daos_fs, dfs_file->daos_file, &wsgl, cur_file_size,
NULL);
if (rc) {
TF_SetStatus(status, TF_RESOURCE_EXHAUSTED, "");
dfs_file->unset_file_size();
Expand Down Expand Up @@ -274,8 +271,8 @@ void NewRandomAccessFile(const TF_Filesystem* filesystem, const char* path,
}
auto random_access_file =
new tf_random_access_file::DFSRandomAccessFile(path, daos->daos_fs, obj);
random_access_file->buffers[0].ReadAsync(
daos->daos_fs, random_access_file->daos_file, 0);
random_access_file->buffers[0].ReadAsync(daos->daos_fs,
random_access_file->daos_file, 0);
file->plugin_file = random_access_file;
TF_SetStatus(status, TF_OK, "");
}
Expand Down
8 changes: 5 additions & 3 deletions tensorflow_io/core/filesystems/dfs/dfs_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,8 @@ int ReadBuffer::ReadAsync(dfs_t* daos_fs, dfs_obj_t* file, const size_t off) {
assert(rc == 0);
rc = daos_event_init(event, eqh, nullptr);
assert(rc == 0);
event->ev_error = dfs_read(daos_fs, file, &rsgl, buffer_offset, &read_size, event);
event->ev_error =
dfs_read(daos_fs, file, &rsgl, buffer_offset, &read_size, event);
return 0;
}

Expand All @@ -571,8 +572,9 @@ int ReadBuffer::CopyData(char* ret, const size_t ret_offset, const size_t off,
}

int64_t ReadBuffer::CopyFromCache(char* ret, const size_t ret_offset,
const size_t off, const size_t n,
const daos_size_t file_size, TF_Status* status) {
const size_t off, const size_t n,
const daos_size_t file_size,
TF_Status* status) {
size_t read_size;
read_size = off + n > file_size ? file_size - off : n;
read_size = off + read_size > buffer_offset + buffer_size
Expand Down
4 changes: 2 additions & 2 deletions tensorflow_io/core/filesystems/dfs/dfs_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,8 @@ class ReadBuffer {
const size_t n);

int64_t CopyFromCache(char* ret, const size_t ret_offset, const size_t off,
const size_t n, const daos_size_t file_size,
TF_Status* status);
const size_t n, const daos_size_t file_size,
TF_Status* status);

private:
size_t id;
Expand Down

0 comments on commit d02b996

Please sign in to comment.