Merge pull request #38 from startin-blox/main

General update

.env.example

@@ -0,0 +1,36 @@
+# DBS RELATED CONFIGS
+APP_NAME=dbs
+PYTHONUNBUFFERED=1
+DJANGO_LOG_LEVEL=DEBUG
+
+# FILECOIN RELATED CONFIGS
+DBS_URL='http://127.0.0.1:8000'
+LOCATION_URL='127.0.0.1'
+PRIVATE_KEY='8837837383'
+LIGHTHOUSE_API_TOKEN='xyv02u242vfvvdgdv24874nnnnndg'
+USER_SQL='root'
+PASSWORD_SQL='root'
+DATABASE_SQL='filecoin_dbs'
+HOST_SQL='127.0.0.1'
+DATANAME_SQL='quote'
+MUMBAI_RPC='https://polygon-mumbai.blockpi.network/v1/rpc/public'
+
+# ARWEAVE RELATED CONFIGS
+ACCEPTED_PAYMENTS='ethereum,matic'
+NODE_RPC_URIS='default,default'
+BUNDLR_URI='https://devnet.bundlr.network'
+PORT='8081'
+# Needs to be provisioned with Matic and GoerliEth
+PRIVATE_KEY='0000000000000000000000000000000000000000000000000000000000000000'
+SQLITE_DB_PATH='db'
+REGISTRATION_INTERVAL=60000
+DBS_URI='http://localhost:8000'
+SELF_URI='http://localhost'
+IPFS_GATEWAY='https://cloudflare-ipfs.com/ipfs/'
+ARWEAVE_GATEWAY='https://arweave.net/'
+MAX_UPLOAD_SIZE='1099511627776'
+# Needs to be provisioned with Matic/WMatic and GoerliETH/WETH
+TEST_PRIVATE_KEY='0000000000000000000000000000000000000000000000000000000000000000'
+ENABLE_EXPENSIVE_TESTS='false'
+CHAIN_ID='80001'
+TOKEN_ADDRESS='0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889'

.github/workflows/django.yml

@@ -16,6 +16,9 @@ jobs:
         python-version: [3.9]
 
     steps:
+    - name: Start Postgresql service
+      run: |
+        sudo systemctl start postgresql.service
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
@@ -31,6 +34,10 @@ jobs:
       run: |
         python server/manage.py migrate
     - name: Run Tests
+      env:
+        PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
+        TEST_PRIVATE_KEY: ${{ secrets.TEST_PRIVATE_KEY }}
+        TOKEN_ADDRESS: ${{ secrets.TOKEN_ADDRESS }}
       run: |
         cd server
         python manage.py test

.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 venv
 server/db.sqlite3
+.env

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "services/dbs_filecoin"]
+	path = services/dbs_filecoin
+	url = https://github.com/oceanprotocol/dbs_filecoin.git
+[submodule "services/dbs_arweave"]
+	path = services/dbs_arweave
+	url = https://github.com/oceanprotocol/dbs_arweave.git

Dockerfile

@@ -10,10 +10,7 @@ RUN pip install -r requirements.txt
 
 # copy project
 COPY . /usr/src/app
-
 WORKDIR /usr/src/app/server
-RUN python manage.py migrate
+
 EXPOSE 8000
-ENV DEBUG=False
-ENV SECRET_KEY='xx'
-CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]

README.md

@@ -33,8 +33,15 @@ You can navigate the API using the url `http://myserver.com/docs/` (Swagger base
 You can also just build the available Dockerfile and run the server from there using the following commands:
 
 ```sh
+git submodule init
+git submodule update
 docker build -t ocean-dbs .
-docker run -p 8000:8000 ocean-dbs
+```
+
+Then you copy/paste the content of .env.example into .env (at the root of the project, and validate that everything is fine by running:
+
+```sh
+docker compose up
 ```
 
 ## Support
@@ -54,6 +61,6 @@ Thanks to the $OCEAN community for the funding and the $OCEAN core team for the
 ## License
 This project is delivered under an Apache License
 
-## Additional projects
+## Associated projects
  - DBS Filecoin microservice (https://github.com/oceanprotocol/dbs_filecoin)
  - DBS Arweave microservice (https://github.com/oceanprotocol/dbs_arweave)

docker-compose.yml

@@ -0,0 +1,52 @@
+version: "3.9"
+services:
+  dbs:
+    container_name: ${APP_NAME}_dbs
+    build: .
+    network_mode: host
+    environment:
+      - PYTHONUNBUFFERED=1
+      - DJANGO_LOG_LEVEL=DEBUG=value
+    volumes:
+      - .:/usr/src/app
+    command:
+      - /bin/sh
+      - '-c'
+      - '/usr/src/app/server/docker-entrypoint.sh'
+  mariadb:
+    container_name: ${APP_NAME}_mariadb
+    image: mariadb
+    restart: always
+    environment:
+      MARIADB_ROOT_PASSWORD: root
+      MARIADB_USER: test
+      MARIADB_PASS: pass
+      MARIADB_DATABASE: filecoin_dbs
+      MARIADB_HOST: 127.0.0.1
+    healthcheck:
+      test: mariadb --user=root --password=root --silent --execute "use filecoin_dbs;"
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    # volumes:
+    #   - ./mounts/mariadb:/var/lib/mariadb
+    network_mode: host
+  dbs_filecoin:
+    container_name: ${APP_NAME}_filecoin
+    build: ./services/dbs_filecoin/
+    env_file: .env # Used to provide the filecoin env variables
+    depends_on:
+      mariadb:
+        condition: service_healthy
+    network_mode: host
+  dbs_arweave:
+    container_name: ${APP_NAME}_arweave
+    build: ./services/dbs_arweave/
+    env_file: .env
+    network_mode: host
+  ipfs:
+    image: ipfs/kubo
+    network_mode: host
+  ipfs-cluster:
+    image: ipfs/ipfs-cluster
+    network_mode: host

examples/file1.txt

@@ -0,0 +1 @@
+e

examples/file2.txt

@@ -0,0 +1 @@
+e

requirements.txt

@@ -5,53 +5,72 @@ APScheduler==3.9.1.post1
 asgiref==3.5.2
 async-timeout==4.0.2
 attrs==22.1.0
-backports.zoneinfo;python_version<="3.9"
+backports.zoneinfo==0.2.1
 base58==2.1.1
+bitarray==2.7.0
 certifi==2022.9.24
 charset-normalizer==2.1.1
+cytoolz==0.12.1
 Django==4.1.2
-django-ipfs-storage==0.0.4
+django-environ==0.9.0
 djangorestframework==3.14.0
 drf-spectacular==0.24.2
+eth-abi==2.2.0
+eth-account==0.5.9
+eth-hash==0.3.3
+eth-keyfile==0.5.1
+eth-keys==0.3.4
+eth-rlp==0.2.1
+eth-typing==2.3.0
+eth-utils==1.10.0
 factory-boy==3.2.1
 Faker==15.1.1
 flake8==5.0.4
 frozenlist==1.3.3
+hexbytes==0.3.0
 idna==3.4
 importlib-resources==5.10.0
 inflection==0.5.1
-ipfsapi==0.4.4
 jsonschema==4.16.0
+lru-dict==1.1.8
 mccabe==0.7.0
 mock==4.0.3
 morphys==1.0
 multiaddr==0.0.9
 multidict==6.0.2
 netaddr==0.8.0
-pkgutil-resolve-name==1.3.10
+parsimonious==0.8.1
+pkgutil_resolve_name==1.3.10
+protobuf==3.19.5
 psycopg2-binary==2.8.6
 py-multibase==1.0.3
 py-multicodec==0.2.1
 py-multiformats-cid==0.4.4
 py-multihash==2.0.1
 pycodestyle==2.9.1
+pycryptodome==3.17
 pyflakes==2.5.0
 pyrsistent==0.18.1
 python-baseconv==1.2.2
 python-dateutil==2.8.2
+python-dotenv==0.21.1
 pytz==2022.5
 pytz-deprecation-shim==0.1.0.post0
 PyYAML==6.0
 requests==2.28.1
 responses==0.22.0
+rlp==2.0.1
 six==1.16.0
 sqlparse==0.4.3
 toml==0.10.2
+toolz==0.12.0
 types-toml==0.10.8.1
 tzdata==2022.6
 tzlocal==4.2
 uritemplate==4.1.1
 urllib3==1.26.12
 varint==1.0.2
+web3==5.31.3
+websockets==9.1
 yarl==1.8.1
 zipp==3.9.0

server/docker-entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Collect static files
+echo "Collect static files"
+python manage.py collectstatic --noinput
+
+# Apply database migrations
+echo "Apply database migrations"
+python manage.py migrate
+
+# Start server
+echo "Starting server"
+python manage.py runserver 0.0.0.0:8000

server/oceandbs/fixtures/storages.json

@@ -2,19 +2,27 @@
   {
     "model": "oceandbs.paymentmethod",
     "pk": 1,
-    "fields": { "chainId": "1", "storage": 1 }
+    "fields": {
+      "chainId": "80001",
+      "storage": 1,
+      "rpcEndpointUrl": "https://rpc-mumbai.maticvigil.com/"
+    }
   },
   {
     "model": "oceandbs.paymentmethod",
     "pk": 2,
-    "fields": { "chainId": "polygon_chain_id", "storage": 1 }
+    "fields": {
+      "chainId": "135",
+      "storage": 1,
+      "rpcEndpointUrl": "https://rpc-mumbai.maticvigil.com/"
+    }
   },
   {
     "model": "oceandbs.acceptedtoken",
     "pk": 1,
     "fields": {
       "title": "OCEAN",
-      "value": "0xOCEAN_on_MAINNET",
+      "value": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889",
       "paymentMethod": 1
     }
   },
@@ -32,7 +40,7 @@
     "pk": 3,
     "fields": {
       "title": "OCEAN",
-      "value": "0xOCEAN_on_POLYGON",
+      "value": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889",
       "paymentMethod": 2
     }
   },
@@ -54,16 +62,27 @@
       "url": "https://filecoin.org/"
     }
   },
+  {
+    "model": "oceandbs.payment",
+    "pk": 1,
+    "fields": {
+      "paymentMethod": 1,
+      "userAddress": "0xCC866199C810B216710A3F3714d35920C343a8CD",
+      "tokenAddress": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889"
+
+    }
+  },
   {
     "model": "oceandbs.quote",
     "pk": 1,
     "fields": {
       "quoteId": "123565",
       "storage": 1,
       "duration": 2380293823,
-      "payment": null,
-      "tokenAddress": "0xOCEAN_on_MAINNET",
-      "approveAddress": "0x123",
+      "payment": 1,
+      "tokenAmount": 500,
+      "tokenAddress": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889",
+      "approveAddress": "0xAFcE990754C38Be5E0C341707B2A162C4e67547B",
       "status": null
     }
   }

server/oceandbs/migrations/0020_alter_acceptedtoken_paymentmethod_and_more.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.1.2 on 2023-01-18 16:31
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('oceandbs', '0019_remove_file_file_file_cid_file_title_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='acceptedtoken',
+            name='paymentMethod',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='acceptedTokens', to='oceandbs.paymentmethod'),
+        ),
+        migrations.AlterField(
+            model_name='payment',
+            name='paymentMethod',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='payments', to='oceandbs.paymentmethod'),
+        ),
+        migrations.AlterField(
+            model_name='paymentmethod',
+            name='storage',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='paymentMethods', to='oceandbs.storage'),
+        ),
+        migrations.AlterField(
+            model_name='quote',
+            name='expiration',
+            field=models.DateTimeField(default=datetime.datetime(2023, 1, 18, 17, 1, 41, 275788, tzinfo=datetime.timezone.utc)),
+        ),
+        migrations.AlterField(
+            model_name='quote',
+            name='nonce',
+            field=models.DateTimeField(default=datetime.datetime(2023, 1, 11, 16, 31, 41, 275770, tzinfo=datetime.timezone.utc)),
+        ),
+    ]