diff --git a/.VERSION b/.VERSION index 3a852e4f..8358d0c9 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v4.0.0-alpha.26/frontend.zip -alpha_version: v4.0.0-alpha.26 -release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v3.2.1/frontend.zip -release_version: v3.2.1 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.0.0-alpha.14/frontend.zip +alpha_version: v5.0.0-alpha.14 +release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.0.0/frontend.zip +release_version: v5.0.0 diff --git a/.github/workflows/cicd_pipeline.yml b/.github/workflows/cicd_pipeline.yml deleted file mode 100644 index 9642a067..00000000 --- a/.github/workflows/cicd_pipeline.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: "DEV CI/CD Pipeline" - -on: - push: - branches: - - dev - workflow_dispatch: - inputs: - branch: - description: 'Frontend branch' - required: true - default: 'release' - type: choice - options: - - release - - alpha - version: - description: 'Current frontend version' - required: true - type: string - name: - description: 'Frontend app name' - required: false - assets_url: - description: 'Frontend assets url' - required: true - type: string - changelog: - description: 'Frontend changelog' - required: false - type: string -jobs: - test: - strategy: - fail-fast: false - matrix: - python-version: [3.7] - poetry-version: ["1.2.2"] - node-version: [16] - os: [ubuntu-20.04] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Resolve frontend - run: | - sh ./scripts/resolve_frontend.sh alpha $FRONTEND_VERSION $FRONTEND_ASSET_URL - echo "PROVIDED_VERSION=0.7.${{ github.run_number }}" >> $GITHUB_ENV - env: - FRONTEND_VERSION: ${{ inputs.version }} - FRONTEND_ASSET_URL: ${{ inputs.assets_url }} - CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }} - - - name: Inject backend info into frontend - uses: satackey/action-js-inline@v0.0.2 - id: InjectBackend - with: - required-packages: '@iarna/toml' - script: | - const fs = require('fs'); - const path = require('path'); - const toml = require('@iarna/toml'); - const rootPath = path.join(process.cwd(), ''); - - console.log('rootPath', rootPath); - - try { - const projectInfo = toml.parse( - fs.readFileSync(path.join(rootPath, 'pyproject.toml'), 'utf8') - ); - const backendInfo = { - version: process.env.PROVIDED_VERSION || projectInfo.tool.poetry.version, - name: projectInfo.tool.poetry.name || 'LabelU', - build_time: new Date().toISOString(), - commit: process.env.GITHUB_SHA, - }; - - const code = ` - (function () { - window.__backend = ${JSON.stringify(backendInfo, null, 2)}; - })(); - `; - - fs.writeFileSync( - path.join(rootPath, 'labelu/internal/statics/backend_version.js'), - code, - 'utf-8' - ); - - console.log('Update backend_version.js success!'); - } catch (e) { - console.error(e); - process.exit(1); - } - - - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} - - name: Install dependencies - run: poetry install --without dev - - name: Run tests - run: poetry run pytest --cov=./ --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./coverage.xml - verbose: true - - name: Manage version - run: | - sed -i "s/^version[ ]*=.*/version = '${PROVIDED_VERSION}'/" pyproject.toml - - name: Publish - env: - TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} - run: | - poetry config pypi-token.testpypi $TEST_PYPI_TOKEN - poetry publish --build --skip-existing -r testpypi - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - registry: https://docker.shlab.tech - username: ${{ secrets.ALI_DOCKERHUB_USERNAME }} - password: ${{ secrets.ALI_DOCKERHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile - push: true - tags: docker.shlab.tech/public/labelu:0.7.${{ github.run_number }}-dev - - name: Commit .VERSION pyproject.toml files to current branch - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "Update .VERSION and pyproject.toml [skip ci]" - file_pattern: '.VERSION pyproject.toml' - - - name: Webhook message - uses: joelwmale/webhook-action@master - with: - url: ${{ secrets.WEBHOOK_URL }} - headers: '{"Content-Type": "application/json"}' - body: '{"msgtype":"markdown","markdown":{"content":"# LabelU@${{ env.PROVIDED_VERSION }}(test) is Released 🎉\n \nCheck it out now \ud83d\udc49\ud83c\udffb [v${{ env.PROVIDED_VERSION }}](https://test.pypi.org/project/labelu/#files) \n \n ## Changelog \n \n${{ inputs.changelog || github.event.head_commit.message }}"}}' - diff --git a/.github/workflows/main_cicd_pipeline.yml b/.github/workflows/main_cicd_pipeline.yml deleted file mode 100644 index bb5a0859..00000000 --- a/.github/workflows/main_cicd_pipeline.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: "main CI/CD Pipeline" - -on: - push: - branches: ["main"] - workflow_dispatch: - inputs: - branch: - description: 'Frontend branch' - required: true - default: 'release' - type: choice - options: - - release - - alpha - version: - description: 'Current frontend version' - required: true - type: string - name: - description: 'Frontend app name' - required: false - assets_url: - description: 'Frontend assets url' - required: true - type: string - changelog: - description: 'Frontend changelog' - required: false - type: string -jobs: - test: - strategy: - fail-fast: false - matrix: - python-version: [3.7] - poetry-version: ["1.2.2"] - node-version: [16] - os: [ubuntu-20.04] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Resolve frontend - run: | - sh ./scripts/resolve_frontend.sh alpha $FRONTEND_VERSION $FRONTEND_ASSET_URL - echo "PROVIDED_VERSION=0.7.${{ github.run_number }}" >> $GITHUB_ENV - env: - FRONTEND_VERSION: ${{ inputs.version }} - FRONTEND_ASSET_URL: ${{ inputs.assets_url }} - FRONTEND_CHANGELOG: ${{ inputs.changelog }} - BACKEND_CHANGELOG: ${{ github.event.head_commit.message }} - CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }} - - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} - - name: Install dependencies - run: poetry install --without dev - - name: Run tests - run: poetry run pytest --cov=./ --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./coverage.xml - verbose: true - - name: Manage version - run: | - sed -i "s/^version[ ]*=.*/version = '${PROVIDED_VERSION}'/" pyproject.toml - - name: Publish - env: - TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} - run: | - poetry config pypi-token.testpypi $TEST_PYPI_TOKEN - poetry publish --build --skip-existing -r testpypi - - name: Commit .VERSION to current branch - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "Update .VERSION [skip ci]" - file_pattern: .VERSION - - name: Webhook message - uses: joelwmale/webhook-action@master - with: - url: ${{ secrets.WEBHOOK_URL }} - headers: '{"Content-Type": "application/json"}' - body: '{"msgtype":"markdown","markdown":{"content":"# LabelU@${{ env.PROVIDED_VERSION }}(test) is Released 🎉\n \nCheck it out now \ud83d\udc49\ud83c\udffb [v${{ env.PROVIDED_VERSION }}](https://test.pypi.org/project/labelu/#files) \n \n ## Changelog \n \n${{ inputs.changelog || github.event.head_commit.message }}"}}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..a4589197 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,185 @@ +name: 'release' + +on: + workflow_dispatch: + inputs: + changelog: + type: string + description: 'The frontend changelog' + required: true + + push: + branches: + - 'main' + - 'alpha' +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + NEXT_VERSION: ${{ steps.dry-run.outputs.NEXT_VERSION }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 20.8.1 + + - name: Semantic Release dry-run + id: dry-run + uses: cycjimmy/semantic-release-action@v4 + with: + dry_run: true + extra_plugins: | + @semantic-release/commit-analyzer + @semantic-release/exec + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + release: + needs: prepare + if: ${{ needs.prepare.outputs.NEXT_VERSION != '' }} + strategy: + fail-fast: false + matrix: + python-version: [3.7] + poetry-version: ['1.2.2'] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Get next version + id: get-next-version + run: | + echo "NEXT_VERSION=$(echo "${{ needs.prepare.outputs.NEXT_VERSION }}")" >> $GITHUB_ENV + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Download frontend + run: | + sh ./scripts/resolve_frontend.sh true + + # ====================== release ====================== + + - name: Set test pip url + if: ${{ github.ref_name == 'alpha' }} + run: | + echo "PYPI_URL=https://test.pypi.org/project/labelu/${{ env.NEXT_VERSION }}" >> $GITHUB_ENV + + - name: Set release pip url + if: ${{ github.ref_name == 'main' }} + run: | + echo "PYPI_URL=https://pypi.org/project/labelu/${{ env.NEXT_VERSION }}" >> $GITHUB_ENV + + - name: Show pypi url + run: | + echo $PYPI_URL + + - name: Inject backend info into frontend + uses: satackey/action-js-inline@v0.0.2 + id: InjectBackend + env: + NEXT_VERSION: ${{ env.NEXT_VERSION }} + with: + required-packages: '@iarna/toml' + script: | + const fs = require('fs'); + const path = require('path'); + const toml = require('@iarna/toml'); + const rootPath = path.join(process.cwd(), ''); + + console.log('rootPath', rootPath); + + try { + const projectInfo = toml.parse( + fs.readFileSync(path.join(rootPath, 'pyproject.toml'), 'utf8') + ); + const backendInfo = { + version: process.env.NEXT_VERSION, + name: projectInfo.tool.poetry.name || 'LabelU', + build_time: new Date().toISOString(), + commit: process.env.GITHUB_SHA, + }; + + const code = ` + (function () { + window.__backend = ${JSON.stringify(backendInfo, null, 2)}; + })(); + `; + + fs.writeFileSync( + path.join(rootPath, 'labelu/internal/statics/backend_version.js'), + code, + 'utf-8' + ); + + console.log('Update backend_version.js success!'); + } catch (e) { + console.error(e); + process.exit(1); + } + + - uses: abatilo/actions-poetry@v2 + with: + poetry-version: ${{ matrix.poetry-version }} + + - name: Install dependencies + run: poetry install --without dev + + - name: Run tests + run: poetry run pytest --cov=./ --cov-report=xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./coverage.xml + verbose: true + + - name: Manage version + run: | + sed -i "s/^version[ ]*=.*/version = '${NEXT_VERSION}'/" pyproject.toml + + - name: Publish to TestPyPi + if: ${{ github.ref_name == 'alpha'}} + env: + TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} + run: | + poetry config pypi-token.testpypi $TEST_PYPI_TOKEN + poetry publish --build --skip-existing -r testpypi + + - name: Publish to PyPi + if: ${{ github.ref_name == 'main'}} + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: | + poetry config pypi-token.pypi $PYPI_TOKEN + poetry publish --build --skip-existing + + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v4 + with: + extra_plugins: | + @semantic-release/commit-analyzer + @semantic-release/release-notes-generator + @semantic-release/exec + @semantic-release/github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # ====================== post release ====================== + + - name: Send webhook message + uses: joelwmale/webhook-action@master + env: + CHANGELOG: ${{ github.event.inputs.changelog != '' && format('## Frontend changelog👇🏻 \n\n{0}\n\n\n', github.event.inputs.changelog) || '' }} + with: + url: ${{ secrets.WEBHOOK_URL }} + headers: '{"Content-Type": "application/json"}' + body: '{"msgtype":"markdown","markdown":{"content":"${{ env.RELEASE_NOTES }}${{ env.CHANGELOG }}Check it out now \ud83d\udc49\ud83c\udffb [v${{ env.NEXT_VERSION }}](${{ env.PYPI_URL }})"}}' diff --git a/.github/workflows/release_cicd_pipeline.yml b/.github/workflows/release_cicd_pipeline.yml deleted file mode 100644 index 01a0328c..00000000 --- a/.github/workflows/release_cicd_pipeline.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: 'release CI/CD Pipeline' - -on: - push: - branches: - - 'release/**' - workflow_dispatch: - inputs: - branch: - description: 'Frontend branch' - required: true - default: 'release' - type: choice - options: - - release - - alpha - version: - description: 'Current frontend version' - required: true - type: string - name: - description: 'Frontend app name' - required: false - assets_url: - description: 'Frontend assets url' - required: true - type: string - changelog: - description: 'Frontend changelog' - required: false - type: string -jobs: - test: - strategy: - fail-fast: false - matrix: - python-version: [3.7] - poetry-version: ['1.2.2'] - node-version: [16] - os: [ubuntu-20.04] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Resolve frontend - run: | - sh ./scripts/resolve_frontend.sh release $FRONTEND_VERSION $FRONTEND_ASSET_URL - echo "PROVIDED_VERSION=${GITHUB_REF##*/v}" >> $GITHUB_ENV - env: - FRONTEND_VERSION: ${{ inputs.version }} - FRONTEND_ASSET_URL: ${{ inputs.assets_url }} - CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }} - - - name: Inject backend info into frontend - uses: satackey/action-js-inline@v0.0.2 - id: InjectBackend - with: - required-packages: '@iarna/toml' - script: | - const fs = require('fs'); - const path = require('path'); - const toml = require('@iarna/toml'); - const rootPath = path.join(process.cwd(), ''); - - console.log('rootPath', rootPath); - - try { - const projectInfo = toml.parse( - fs.readFileSync(path.join(rootPath, 'pyproject.toml'), 'utf8') - ); - const backendInfo = { - version: process.env.PROVIDED_VERSION || projectInfo.tool.poetry.version, - name: projectInfo.tool.poetry.name || 'LabelU', - build_time: new Date().toISOString(), - commit: process.env.GITHUB_SHA, - }; - - const code = ` - (function () { - window.__backend = ${JSON.stringify(backendInfo, null, 2)}; - })(); - `; - - fs.writeFileSync( - path.join(rootPath, 'labelu/internal/statics/backend_version.js'), - code, - 'utf-8' - ); - - console.log('Update backend_version.js success!'); - } catch (e) { - console.error(e); - process.exit(1); - } - - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} - - name: Install dependencies - run: poetry install --without dev - - name: Run tests - run: poetry run pytest --cov=./ --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./coverage.xml - verbose: true - - name: Manage version - run: | - sed -i "s/^version[ ]*=.*/version = '${PROVIDED_VERSION}'/" pyproject.toml - - name: Publish - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - poetry config pypi-token.pypi $PYPI_TOKEN - poetry publish --build --skip-existing - - name: Commit .VERSION to current branch - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: 'Update .VERSION [skip ci]' - file_pattern: .VERSION - - name: Webhook message - uses: joelwmale/webhook-action@master - with: - url: ${{ secrets.WEBHOOK_URL }} - headers: '{"Content-Type": "application/json"}' - body: '{"msgtype":"markdown","markdown":{"content":"# LabelU@${{ env.PROVIDED_VERSION }}(prod) is Released 🎉\n \nCheck it out now \ud83d\udc49\ud83c\udffb [v${{ env.PROVIDED_VERSION }}](https://pypi.org/project/labelu/#files) \n \n ## Changelog \n \n${{ inputs.changelog || github.event.head_commit.message }}"}}' diff --git a/.github/workflows/trigger_from_labelu_kit.yml b/.github/workflows/trigger_from_labelu_kit.yml new file mode 100644 index 00000000..32335ea1 --- /dev/null +++ b/.github/workflows/trigger_from_labelu_kit.yml @@ -0,0 +1,85 @@ +name: 'Updating .version triggered from LabelU-Kit' + +on: + workflow_dispatch: + inputs: + branch: + description: 'Frontend branch' + required: true + default: 'release' + type: choice + options: + - release + - alpha + version: + description: 'Current frontend version' + required: true + type: string + release_type: + description: 'Release type(fix, feat, breaking-change)' + required: true + type: choice + default: 'fix' + options: + - fix + - feat + - breaking-change + changelog: + description: 'Frontend changelog' + required: false + assets_url: + description: 'Frontend assets url' + required: true + type: string +jobs: + update-frontend: + strategy: + fail-fast: false + matrix: + node-version: [20.8.1] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + + permissions: + actions: write + contents: write + issues: write + pull-requests: write + id-token: write + + steps: + # ====================== frontend ====================== + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Update .VERSION file + run: | + sh ./scripts/resolve_frontend.sh ${{ github.ref_name }} $FRONTEND_VERSION $FRONTEND_ASSET_URL + + env: + FRONTEND_VERSION: ${{ inputs.version }} + FRONTEND_ASSET_URL: ${{ inputs.assets_url }} + CURRENT_BRANCH: ${{ github.ref_name }} + + - name: Commit .VERSION file + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '${{ inputs.release_type }}: Upgrade frontend to ${{ inputs.version }}. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/${{ inputs.version }})' + file_pattern: .VERSION + + - name: Prepare changelog + id: set_changelog + run: | + echo "CHANGE_LOG=$(echo "${{ inputs.changelog }}" | sed ':a;N;$!ba;s/\n/\\n/g')" >> $GITHUB_ENV + + + - name: Trigger release workflow + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: release + inputs: | + { + "changelog": "${{ env.CHANGE_LOG }}" + } + \ No newline at end of file diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 00000000..7da0a562 --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,34 @@ +{ + "branches": ["main", { "name": "alpha", "prerelease": true }], + "plugins": [ + [ + { + "parserOpts": { + "noteKeywords": ["BREAKING CHANGE", "BREAKING CHANGES"] + } + } + ], + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + [ + "@semantic-release/github", + { + "assets": [{ "path": "./dist/*.whl", "label": "labelu-${nextRelease.version}-py3.whl" }, { "path": "./dist/*.gz", "label": "labelu-${nextRelease.version}.tar.gz" }] + } + ], + [ + "@semantic-release/git", + { + "assets": ["pyproject.toml"], + "message": "chore(release): ${nextRelease.version} [skip ci] \n\n${nextRelease.notes}" + } + ], + [ + "@semantic-release/exec", + { + "verifyReleaseCmd": "echo \"NEXT_VERSION=${nextRelease.version}\" >> $GITHUB_OUTPUT", + "successCmd": "echo \"RELEASE_NOTES<> $GITHUB_ENV" + } + ] + ] +} \ No newline at end of file diff --git a/docs/GUIDE.md b/docs/GUIDE.md deleted file mode 100644 index aeb92841..00000000 --- a/docs/GUIDE.md +++ /dev/null @@ -1,192 +0,0 @@ -简体中文 | English - -# Concept - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

Concept

- 		-

Description

-
-

Task

- 		-

Task is created to label a dataset

-
-

label

- 		-

Classification marks to be added to an - annotation, such as cat, dog, pedestrian and vehicle

-
-

Annotation

- 		-

Objects generated after a marking - operation, such as a rectangular box or a point

-
-

Attribute

- 		-

Further description of a label, such - as adding the attribute "vehicle occlusion rate is 20%" after - marking an object as a vehicle

-
-

Result

- 		-

Annotation+Label+Attribute, a - complete annotation record

-
-

Template

- 		-

The configuration files (XML, JSON, - Yaml) that can be referenced when configuring labeling tasks

-
- -# Register && log in - -1. Create an account with email and password -2. Log in to LabelU - -# Create a new task - -![image](https://user-images.githubusercontent.com/25022954/208387913-9a4a8205-8dfc-423f-997d-5c6f277ec0eb.png) - -1. Click "New Task" -2. Enter the task name, task description, and task hints, among which the task hints will provide hints and help for the labelers during the labeling process -3. Click "Next" to save the basic configuration information and enter the "Data Import" page - -# Data import - -![image](https://user-images.githubusercontent.com/25022954/208388040-79b49127-adc0-4468-81d6-f78dc6a80a46.png) - -Currently, data import only supports local data import, and supports file or folder import in two ways. -1. Click Upload File or Upload Folder to upload local data -2. Click "Next" to save and enter the label configuration -3. If you want to import data again, you can click "Data Import" on the task home page to enter the data import page to import, and the imported data can be viewed on the task home page - -# Task configuration - -After completing the basic configuration and data import configuration, perform labeling configuration, mainly according to the task scenario to edit the configuration of tools and labels, and configure some other parameters. Currently supports Yaml and visualization two configuration methods, the two methods support linkage, and can preview the home page through the right page - -## Yaml configuration (to be perfected) - -1. Select a template according to the task requirements -2. Customize and modify the Yaml configuration, including tool configuration modification, label configuration modification, other configuration modification, etc. -3. Click Finish - -## Visual configuration - -![image](https://user-images.githubusercontent.com/25022954/208390163-e6b34056-a618-485a-8875-38f99741ee68.png) - -In addition to using the Yaml configuration, you can also choose to use the visual method to configure the annotations. The configuration process is as follows: -1. Select the annotation type, currently only supports pictures, and will support audio and video, point cloud, text and other types in the future -2. Select the annotation tool, which supports simultaneous configuration of multiple tools (drawing box, polygon, punctuation, marking, classification, description) -3. The configuration required for each tool includes tool properties and label configuration - -Configuration instructions for each tool: -|Tool|Instruction| -|---|---| -|rectangle|Minimum size: the minimum width (W) and height (H) of the pull-up box, label configuration: label object classification, including Chinese and English| -|polygon|Line type: including straight lines and curves, number of closing points: including the minimum number of closing points and the maximum number of closing points, edge adsorption: automatically fit the edge of the object after opening, label configuration: label object classification, including Chinese and English| -|point|Upper limit of points: the upper limit of specified points, exceeding the limit cannot be drawn, label configuration: label object classification, including Chinese and English| -|line|Line type: including straight lines and curves, number of closing points: including the minimum number of closing points and the maximum number of closing points, edge adsorption: automatically fit the edge of the object after opening, label configuration: label object classification, including Chinese and English| -|text|Text list: include text description name in Chinese and English, text setting: include maximum number of words and default text| -|classification|Category: Category description in Chinese and English, Options: Category specific options in Chinese and English| - -4. Choose whether to support out-of-target annotations, and configure attributes (currently only text types are supported) -5. Set common labels (when multiple tools share a label system, configuration can be simplified by setting common labels) -6. Preview to view the effect -7. Click Save to enter the task home page - -# Start labeling - -![image](https://user-images.githubusercontent.com/25022954/208390649-cc0bccb1-c509-4623-aeef-44f6649adc4c.png) - -Start labeling tasks according to the task configuration. Each functional area is introduced as follows: -Toolbar: perform operations such as tool selection, tool style switching, undo and redo, display order, etc. -Label bar: After selecting the tool, click to select the label to label -Marking result bar: View and edit the marked results -Annotation subject: including drawing area, picture operation - -Labeling process description: -1. Determine whether the task is invalid. If it is invalid, click Skip to enter the next task -If it is valid, when there are drawing tasks (object detection, semantic segmentation, line labeling, point labeling), the tools and configurations of the drawing tasks are consistent -2. Select Tool -3. Select tab -4. Mark the drawing -5. The attribute information pop-up window will pop up, edit the attribute information, if there is no attribute information, you can leave it blank, click anywhere outside the pop-up window to close. -6. Click the tag result in the tag result management bar on the right to select the corresponding mark in the picture, and switch the tool selected in the toolbar to the tag result tool. Click the [Edit] [Show/Hide] [Delete] button of the tag result to manage it. -If valid, when there are label descriptions and classification tasks -7. Fill in the description and classification results in the result management column on the right -8. Select [Next] to enter the next task -9. Repeat 1~8 until the marking is completed - -For specific labeling operations of each tool, see: - - [Image Labeling](./labeling/label_image.md) - -# Export results - -After the annotation is completed, the annotation result file can be exported in the form of Json. The annotation format is described as follows: - -[LabelU annotation format](./annotation%20format/README.md) diff --git a/docs/GUIDE_zh-CN.md b/docs/GUIDE_zh-CN.md deleted file mode 100644 index 3d209ff4..00000000 --- a/docs/GUIDE_zh-CN.md +++ /dev/null @@ -1,188 +0,0 @@ - 简体中文 | English - -# 概念 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

概念

- 		-

说明

-
-

任务（task）

- 		-

为了对某个数据集进行标注而建立的任务

-
-

标签（label）

- 		-

标注时需要添加的分类标识，比如猫、狗、行人、车辆

-
-

标记（annotation）

- 		-

进行一次标注后生成的对象，比如一个矩形框、一个点

-
-

属性（attribute）

- 		-

对标签的进一步描述，比如将某个物体标为车辆后，添加属性“车辆遮挡率为20%”

-
-

标注结果（result）

- 		-

标记+标签+属性，一条完整的标注记录

-
-

模板（template）

- 		-

对标注任务进行配置时，可参照的配置文件（XML、JSON、Yaml）

-
- -# 注册登录 - -1. 通过邮箱密码创建账号 -2. 登录进入LabelU - -# 新建任务 - -![image](https://user-images.githubusercontent.com/25022954/208387913-9a4a8205-8dfc-423f-997d-5c6f277ec0eb.png) - -1. 点击“新建任务” -2. 输入任务名称、任务描述、任务提示，其中任务提示会在标注过程中为标注人员提供提示帮助 -3. 点击“下一步”保存基础配置信息，并进入“数据导入”页面 - -# 数据导入 - -![image](https://user-images.githubusercontent.com/25022954/208388040-79b49127-adc0-4468-81d6-f78dc6a80a46.png) - -目前数据导入仅支持本地数据导入，支持文件或文件夹导入两种方式。 -1. 点击上传文件或上传文件夹，上传本地数据 -2. 点击“下一步”保存并进入标注配置 -3. 若想再次导入数据，可通过点击任务主页中的“数据导入”进入数据导入页面进行导入，导入的数据可在任务主页中进行查看 - -# 任务配置 - -在完成基础配置和数据导入配置后，进行标注配置，主要根据任务场景进行工具和标签的配置编辑，以及其他一些参数配置。目前支持Yaml和可视化两种配置方式，两种方式支持联动，并可通过右边页面进行标注主页的预览 - -## Yaml配置（待完善） - -1. 根据任务需求选择模板 -2. 自定义修改Yaml配置，包括工具配置修改、标签配置修改、其他配置修改等 -3. 点击完成 - -## 可视化配置 - -![image](https://user-images.githubusercontent.com/25022954/208390163-e6b34056-a618-485a-8875-38f99741ee68.png) - -除采用Yaml方式配置外，也可选择采用可视化方式进行标注配置，配置流程如下： -1. 选择标注类型，目前仅支持图片，后续将支持音视频、点云、文本等类型 -2. 选择标注工具，支持同时配置多种工具（拉框、多边形、标点、标线、分类、描述） -3. 各工具所需配置包括工具属性和标签配置 - -各工具配置说明： -|工具|配置说明| -|---|---| -|拉框|最小尺寸：拉框最小宽度（W）和高度（H），标签配置：标注对象分类，包括中英文| -|多边形|线条类型：包括直线和曲线，闭合点数：包括最小闭合点数和最大闭合点数，边缘吸附：开启后自动贴合物体边缘，标签配置：标注对象分类，包括中英文| -|标点|上限点数：规定点数上限，超过不可画，标签配置：标注对象分类，包括中英文| -|标线|线条类型：包括直线和曲线，闭合点数：包括最小闭合点数和最大闭合点数，边缘吸附：开启后自动贴合物体边缘，标签配置：标注对象分类，包括中英文| -|文本|文本列表：包括文本描述名称中英文，文本设置：包括最大字数和默认文本| -|分类|类别：分类描述中英文，选项：分类的具体选项中英文| - -4. 选择是否支持目标外标注、进行属性配置（目前仅支持文本类型） -5. 设置通用标签（当多工具共用一套标签体系时，可通过设置通用标签简化配置） -6. 预览查看效果 -7. 点击保存，进入任务主页 - -# 开始标注 - -![image](https://user-images.githubusercontent.com/25022954/208390649-cc0bccb1-c509-4623-aeef-44f6649adc4c.png) - -根据任务配置情况开始标注任务，各功能区域介绍如下： -工具栏：进行工具选择、工具样式切换、撤回重做、显示顺序等操作 -标签栏：选择工具后单击选择标签进行标注 -标注结果栏：对标注的结果进行查看和编辑 -标注主体：包括绘图区域、图片操作 - -标注流程说明： -1. 判断任务是否为无效，若无效点击跳过，进入下一任务 -若有效，有绘图任务时（目标检测、语意分割、线标注、点标注），绘图任务时工具和配置的一致 -2. 选择工具 -3. 选择标签 -4. 绘图做标记 -5. 弹出属性信息弹窗，编辑属性信息，若无属性信息可不填，点击弹窗外任意处关闭。 -6. 在右侧标签结果管理栏单击标签结果可选中图片中对应标记，工具栏中选中工具切换为标签结果的工具。点击标签结果的【编辑】【显示/隐藏】【删除】按钮来管理。 -若有效，有标签描述、分类任务时 -7. 在右侧结果管理栏填写描述和分类结果 -8. 选择[下一页]，进入下一个任务 -9. 重复1~8，直到标注完成 - -各工具具体标注操作详见： - - [图像标注](./labeling/label_image.md) - -# 导出结果 - -完成标注后，可将标注结果文件以Json形式导出，标注格式说明如下： - -[LabelU 标注格式](./annotation%20format/README.md) diff --git a/docs/annotation format/README.md b/docs/annotation format/README.md deleted file mode 100644 index 9757b130..00000000 --- a/docs/annotation format/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# 标注格式说明 - -## 通用格式 - -```json -{ - "width": 4368, - "height": 2912, - "valid": true, - "rotate": 0, - "annotations": [ - { - "toolName": "rectTool", - "result": [ - { - "x": 530.7826086956522, - "y": 1149.217391304348, - "width": 1314.7826086956522, - "height": 1655.6521739130435, - "label": "cat", - "valid": true, - "id": "Rp1x6bZs", - "textAttribute": "", - "order": 1 - } - ] - }, - { - "toolName": "polygonTool", - "result": [ - { - "id": "j91grNMP", - "pointList": [ - { - "x": 298.4319526627219, - "y": 155.54201183431957 - }, - { - "x": 246.85325443786985, - "y": 203.09112426035506 - }, - { - "x": 248.46508875739647, - "y": 289.3242603550296 - }, - { - "x": 320.9976331360947, - "y": 345.73846153846154 - } - ], - "valid": true, - "label": "cat", - "order": 1, - "textAttribute": "", - } - ] -} -``` - -## 各工具导出格式说明 - -- [目标检测](./rectTool.md) -- [目标分类](./tagTool.md) -- [物体分割](./polygonTool.md) -- [文本转写](./textTool.md) -- [轮廓线检测](./lineTool.md) -- [关键点检测](./pointTool.md) - - -## Mask 格式导出说明 - -仅分割任务（多边形）可导出 Mask。 - -- 背景色默认为黑色： 0 `（rgb(0,0,0))` -- 语义的唯一性：语义分割的属性标注配置 -- 导出内容: - - Mask - - 彩色图（xx_segmentation.png)：用于结果校验 - - 灰度图（xx_labelTrainIds.png)：用于训练 - - JSON 文件： 表示当前语义与颜色的索引关系 - -```json -[ - { - "attribute": "cat", - "color": "rgb(128,0,0)", - "trainIds": 1 - }, - { - "attribute": "dog", - "color": "rgb(0,128,0)", - "trainIds": 2 - }, - { - "attribute": "bird", - "color": "rgb(128,128,0)", - "trainIds": 3 - }, -] -``` - -| 名称 | 描述 | -| --------- | ------------------------------------------ | -| attribute | 当前语义 | -| color | 当前语义颜色 | -| trainIds | 训练使用的 ID （灰度值 1 - N，0 表示背景） | diff --git a/docs/annotation format/lineTool.md b/docs/annotation format/lineTool.md deleted file mode 100644 index c4ee1342..00000000 --- a/docs/annotation format/lineTool.md +++ /dev/null @@ -1,55 +0,0 @@ -# 线条(lineTool) - -## 格式说明 - -```json -{ - "width": 1024, - "height": 681, - "rotate": 0, - "valid": true, - "annotations": [ - { - "toolName": "lineTool", - "result": [ - { - "pointList": [ - { - "x": 341.95147928994083, - "y": 138.61775147928998, - "id": "TlfF2xY4" - }, - { - "x": 263.7775147928994, - "y": 292.54792899408284, - "id": "eKivOQt6" - }, - { - "x": 428.9905325443787, - "y": 409.4059171597633, - "id": "5L6zZWRM" - }, - { - "x": 763.4461538461538, - "y": 384.42248520710064, - "id": "IXuGhnQw" - }, - { - "x": 781.9822485207101, - "y": 349.7680473372781, - "id": "Oiy6pk9d" - } - ], - "id": "rI89mHsg", - "valid": true, - "order": 1, - "label": "class-6U", - "textAttribute": "xxxxxd" - } - ] - } - ] -} -``` - -
名称类型是否必须默认值备注其他信息
resultobject []
必须
线条列表，数量应与标注对象一致

item 类型: object

pointListobject []
必须
点列表

item 类型: object

idstring
必须
线条ID

mock: @string

validboolean
必须
是否有效

mock: @boolean

ordernumber
必须
序号

mock: @natural

labelstring
非必须
标签

mock: @string

textAttributestring
非必须
文本

mock: @string

diff --git a/docs/annotation format/pointTool.md b/docs/annotation format/pointTool.md deleted file mode 100644 index f1bbb13e..00000000 --- a/docs/annotation format/pointTool.md +++ /dev/null @@ -1,30 +0,0 @@ -# 标点(pointTool) - -## 格式说明 - -```json -{ - "width": 1024, - "height": 681, - "rotate": 0, - "valid": true, - "annotations": [ - { - "toolName": "pointTool", - "result": [ - { - "x": 300.84970414201183, - "y": 298.18934911242604, - "valid": true, - "id": "ts5DBxDI", - "order": 1, - "label": "class-1Q", - "textAttribute": "我是标点" - } - ] - } - ] -} -``` - -
名称类型是否必须默认值备注其他信息
resultobject []
必须

item 类型: object

xnumber
必须
点在X轴坐标
ynumber
必须
点在Y轴坐标
validboolean
必须
有效性
idstring
必须
ID
ordernumber
必须
标点顺序
labelstring
非必须
标签
textAttributestring
非必须
文本属性
diff --git a/docs/annotation format/polygonTool.md b/docs/annotation format/polygonTool.md deleted file mode 100644 index 37da1d68..00000000 --- a/docs/annotation format/polygonTool.md +++ /dev/null @@ -1,495 +0,0 @@ -# 物体分割(polygonTool) - -## 格式说明 - -```json -{ - "width": 1024, - "height": 681, - "rotate": 0, - "valid": true, - "annotations": [ - { - "toolName": "polygonTool", - "result": [ - { - "id": "j91grNMP", - "pointList": [ - { - "x": 298.4319526627219, - "y": 155.54201183431957 - }, - { - "x": 246.85325443786985, - "y": 203.09112426035506 - }, - { - "x": 248.46508875739647, - "y": 289.3242603550296 - }, - { - "x": 320.9976331360947, - "y": 345.73846153846154 - } - ], - "valid": true, - "label": "class-BX", - "order": 1, - "textAttribute": "", - } - ] - } - ] -} -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

名称

- 		-

类型

- 		-

是否必须

- 		-

默认值

- 		-

备注

- 		-

其他信息

-
-

result

- 		-

object []

- 		-

必须

- 		-

item 类型: object

-
-

id

- 		-

string

- 		-

必须

- 		-

当前id

-
-

pointList

- 		-

object []

- 		-

必须

- 		-

多边形点集

- 		-

item 类型: object

-
-

x

- 		-

number

- 		-

必须

- 		-

点的 x 坐标

-
-

y

- 		-

number

- 		-

必须

- 		-

点的 y 坐标

-
-

valid

- 		-

boolean

- 		-

必须

- 		-

当前多边形有无效性

-
-

order

- 		-

number

- 		-

必须

- 		-

当前多边形的序号

-
-

textAttribute

- 		-

string

- 		-

非必须

- 		-

多边形的文本属性

-
-

label

- 		-

string

- 		-

非必须

- 		-

多边形配置的属性

-
diff --git a/docs/annotation format/rectTool.md b/docs/annotation format/rectTool.md deleted file mode 100644 index 16ec71f3..00000000 --- a/docs/annotation format/rectTool.md +++ /dev/null @@ -1,384 +0,0 @@ -# 目标检测(rectTool) - -## 格式说明 - -```json -{ - "width": 4368, - "height": 2912, - "valid": true, - "rotate": 0, - "annotations": [ - { - "toolName": "rectTool", - "result": [ - { - "x": 530.7826086956522, - "y": 1149.217391304348, - "width": 1314.7826086956522, - "height": 1655.6521739130435, - "attribute": "", - "valid": true, - "id": "Rp1x6bZs", - "textAttribute": "", - "order": 1 - } - ] - } - ] -} -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

名称

- 		-

类型

- 		-

是否必须

- 		-

默认值

- 		-

备注

- 		-

其他信息

-
-

result

- 		-

object - []

- 		-

必须

- 		-

item - 类型: object

-
-

id

- 		-

string

- 		-

必须

- 		-

矩形框 id

-
-

x

- 		-

number

- 		-

必须

- 		-

矩形框起始点横坐标（左上角）

-
-

y

- 		-

number

- 		-

必须

- 		-

矩形框起始点纵坐标（左上角）

-
-

width

- 		-

number

- 		-

必须

- 		-

矩形框宽度

-
-

height

- 		-

number

- 		-

必须

- 		-

矩形框高度

-
-

valid

- 		-

boolean

- 		-

必须

- 		-

有无效性

-
-

order

- 		-

number

- 		-

必须

- 		-

矩形框顺序

-
-

textAttribute

- 		-

string

- 		-

必须

- 		-

文本属性

-
-

label

- 		-

string

- 		-

必须

- 		-

标签

-
diff --git a/docs/annotation format/tagTool.md b/docs/annotation format/tagTool.md deleted file mode 100644 index 44037d51..00000000 --- a/docs/annotation format/tagTool.md +++ /dev/null @@ -1,30 +0,0 @@ -# 图像分类(tagTool) - -## 格式说明 - - - -```json -{ - "width": 1024, - "height": 576, - "rotate": 0, - "valid": true, - "annotations": [ - { - "toolName": "tagTool", - "result": [ - { - "id": "CXHF5QUM", - "result": { - "class1": "option1-2;option1", - "class-Xn": "option2-1" - } - } - ] - } - ] -} -``` - - diff --git a/docs/annotation format/textTool.md b/docs/annotation format/textTool.md deleted file mode 100644 index 39d2a222..00000000 --- a/docs/annotation format/textTool.md +++ /dev/null @@ -1,215 +0,0 @@ -# 文本(textTool) - -## 格式说明 - -```json -{ - "width": 1024, - "height": 681, - "rotate": 0, - "valid": true, - "annotations": [ - { - "toolName": "textTool", - "result": [ - { - "value": { - "text": "textValue" - }, - "id": "ULNah0Wf", - } - } - ] - } - ] -} -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

名称

- 		-

类型

- 		-

是否必须

- 		-

默认值

- 		-

备注

- 		-

其他信息

-
-

result

- 		-

object []

- 		-

必须

- 		-

图片上的结果列表，数量应与标注对象一致

- 		-

item 类型: object

-
-

id

- 		-

string

- 		-

必须

- 		-

文本内容的id

-
-

value

- 		-

object

- 		-

必须

- 		-

文本内容

-
diff --git a/docs/labeling/label_image.md b/docs/labeling/label_image.md deleted file mode 100644 index d70b8a2e..00000000 --- a/docs/labeling/label_image.md +++ /dev/null @@ -1,46 +0,0 @@ -# 图像标注 -## 物体检测-拉框 -拉框标注是目前应用最广泛的一种图像标注方法，能够以一种相对简单便捷的方式在图像或视频数据中，迅速框定住指定目标对象。 -### 标注操作： -1. 选中拉框工具并配置标签，如轿车、公交车。 -2. 单击鼠标左键标注第一点，画出框的范围之后再次单击左键，即可绘制出框。 -3. 右键选中框之后，可以调整框的大小、有效性，也可以删除已标注的框。 -4. 右侧工具栏可调整拉框颜色以及边框、填充透明度等操作。 - -## 图像分割-多边形 -多边形标注是指在静态图片中，使用多边形框，标注出不规则的目标物体，相对于拉框标注，多边形标注能够更精准地识别目标，同时对于不规则物体，也更有针对性。 -### 标注操作： -1. 选中多边形工具并配置标签，如猫、狗。 -2. 单击鼠标左键标注起始点，随后沿目标边缘多次单击左键，以此类推在围绕目标边缘右键连接最接近起始点后，即可绘制出多边形框。 -3. 右键选中框之后，可以调整目标边缘关键点、线段、目标有效性，也可以删除已标注的多边形框。 -4. 右侧工具栏可调整多边形颜色以及边框、填充透明度等操作。 - -## 人体姿态-标点 -关键点标注是通过单个或多个连续的关键点确定巨大和微小物体或人体姿态的形状变化，通常用于统计模型以及姿势或面部识别模型。 -### 标注操作： -1. 选中标点工具并配置标签，如人体姿态14个关键点，包括头、脖子、左肩、右肩、左手肘、右手肘、左腕、右腕、左髋、右髋、左膝、右膝、左脚踝、右脚踝。 -2. 单击鼠标左键标注指定关键点。 -3. 右键选中点之后，可以调整点的位置、属性，也可以删除已标注的点。 -4. 右侧工具栏可调整标点颜色、大小。 - -## 车道线标注-标线 -线条工具目前主要应用在自动驾驶、人体轮廓姿态等标注场景，进行车道线、人体轮廓等识别。此外还应用于表格线的标注识别，帮助发票和检查报告等含有表格线的印刷纸进行快速识别。 -### 标注操作： -1. 选中线条工具并配置标签，如车道线。 -2. 单击鼠标左键标注起始点，再次点击1+N后右键为终止点。 -3. 操作Shift+左键则为垂直或水平线。 -4. 右侧工具栏可调整线条颜色、大小。 - -## 快捷键 - -

- -

- -

- -

- -

- -

diff --git a/labelu/alembic_labelu/alembic_labelu_tools.py b/labelu/alembic_labelu/alembic_labelu_tools.py index 8d3508d7..0ed629e8 100644 --- a/labelu/alembic_labelu/alembic_labelu_tools.py +++ b/labelu/alembic_labelu/alembic_labelu_tools.py @@ -4,6 +4,29 @@ from sqlalchemy import engine_from_config from sqlalchemy.engine import reflection +def table_exist(table_name): + """check table is not exist + + Args: + table_name (string): the name of table + + Returns: + bool: true or false, whether the table_name exists + """ + config = op.get_context().config + engine = engine_from_config( + config.get_section(config.config_ini_section), prefix="sqlalchemy." + ) + insp = reflection.Inspector.from_engine(engine) + table_exist = False + + for table in insp.get_table_names(): + if table_name not in table: + continue + table_exist = True + break + + return table_exist def column_exist_in_table(table_name, column_name): """check column is not exist in table diff --git a/labelu/alembic_labelu/versions/1b174ca5159a_update_fields.py b/labelu/alembic_labelu/versions/1b174ca5159a_update_fields.py new file mode 100644 index 00000000..d5b3d323 --- /dev/null +++ b/labelu/alembic_labelu/versions/1b174ca5159a_update_fields.py @@ -0,0 +1,88 @@ +"""update pointList => points + +Revision ID: 1b174ca5159a +Revises: 0145db0fec34 +Create Date: 2024-02-01 19:26:24.048019 + +""" +import json +from alembic import context, op +from sqlalchemy import update +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import sessionmaker + +Base = automap_base() + +def update_fields(sample_data: dict) -> dict: + """update the field pointList to points in sample_data""" + # get the result of sample_data + annotated_result = sample_data.get("result") + annotated_result = json.loads(annotated_result) + # update the field pointList to points + for sample_tool, sample_tool_results in annotated_result.items(): + if sample_tool.endswith("Tool") and isinstance(sample_tool_results, dict): + for sample_tool_result in sample_tool_results.get("result", []): + keys = sample_tool_result.keys() + # replace pointList to points + if "pointList" in keys: + sample_tool_result["points"] = sample_tool_result.pop("pointList") + # replace attribute to label + if "attribute" in keys: + sample_tool_result["label"] = sample_tool_result.pop("attribute") + return annotated_result + + +# revision identifiers, used by Alembic. +revision = '1b174ca5159a' +down_revision = '0145db0fec34' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Replace pointList with points in result table""" + # get the current connection + bind = op.get_bind() + # Reflect ORM models from the database + Base.prepare(autoload_with=bind, reflect=True) + task_sample = Base.classes.task_sample + # make a session + Session = sessionmaker(bind=bind) + session = Session() + + with context.begin_transaction(): + task_items = session.execute( + 'SELECT id, config FROM task WHERE config IS NOT NULL AND config != ""' + ) + + for task_item in task_items: + task_id = task_item[0] + + sample_items = session.execute( + f"SELECT id, data, annotated_count FROM task_sample WHERE task_id={task_id}" + ) + + for sample_item in sample_items: + sample_id = sample_item[0] + sample_data_item = json.loads(sample_item[1]) + if sample_item[2] > 0: + sample_annotated_result = update_fields( + sample_data_item + ) + sample_data_item["result"] = json.dumps( + sample_annotated_result, ensure_ascii=False + ) + # display unicode as the corresponding Chinese + else: + pass + sample_annotated_item_str = json.dumps( + sample_data_item, ensure_ascii=False + ) + op.execute( + update(task_sample) + .where(task_sample.id == sample_id) + .values({task_sample.data: sample_annotated_item_str}) + ) + +def downgrade() -> None: + pass diff --git a/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py new file mode 100644 index 00000000..94dc6250 --- /dev/null +++ b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py @@ -0,0 +1,163 @@ +"""add file and pre_annotation + +Revision ID: bc8fcb35b66b +Revises: 1b174ca5159a +Create Date: 2024-02-07 15:58:30.618151 + +""" +import imp +import json +import os + +from alembic import context, op +import sqlalchemy as sa +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import sessionmaker +from labelu.internal.common.config import settings + +Base = automap_base() + + +# revision identifiers, used by Alembic. +revision = 'bc8fcb35b66b' +down_revision = '1b174ca5159a' +branch_labels = None +depends_on = None + +# import alembic_labelu_tools from the absolute path +alembic_labelu_tools = imp.load_source( + "alembic_labelu_tools", + ( + os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(__file__))), + "alembic_labelu_tools.py", + ) + ), +) + +def upgrade() -> None: + bind = op.get_bind() + Base.prepare(autoload_with=bind, reflect=True) + task_sample = Base.classes.task_sample + # make a session + Session = sessionmaker(bind=bind) + session = Session() + + with context.begin_transaction(): + # Create a new table task_pre_annotation + if not alembic_labelu_tools.table_exist("task_pre_annotation"): + op.create_table( + "task_pre_annotation", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True, index=True), + sa.Column("task_id", sa.Integer, sa.ForeignKey("task.id"), index=True), + sa.Column("file_id", sa.Integer, sa.ForeignKey("task_attachment.id"), index=True), + # json 字符串 + sa.Column("data", sa.Text, comment="task sample pre annotation result"), + sa.Column("created_by", sa.Integer, sa.ForeignKey("user.id"), index=True), + sa.Column("updated_by", sa.Integer, sa.ForeignKey("user.id")), + sa.Column( + "created_at", + sa.DateTime, + default=sa.func.now(), + comment="Time a task sample result was created", + ), + sa.Column( + "updated_at", + sa.DateTime, + default=sa.func.now(), + onupdate=sa.func.now(), + comment="Last time a task sample result was updated", + ), + sa.Column( + "deleted_at", + sa.DateTime, + index=True, + comment="Task delete time", + ), + ) + # Update the task_sample table + if not alembic_labelu_tools.column_exist_in_table( + "task_sample", "file_id" + ): + with op.batch_alter_table('task_sample', recreate="always") as batch_op: + batch_op.add_column( + sa.Column( + "file_id", + sa.Integer(), + sa.ForeignKey("task_attachment.id", name="fk_file_id"), + index=True, + comment="file id", + ), + ) + + # Update the task_attachment table + if not alembic_labelu_tools.column_exist_in_table("task_attachment", "filename"): + with op.batch_alter_table("task_attachment", recreate="always") as batch_op_task_attachment: + batch_op_task_attachment.add_column( + sa.Column( + "filename", + sa.String(256), + comment="file name", + ), + ) + batch_op_task_attachment.add_column( + sa.Column( + "url", + sa.String(256), + comment="file url", + ), + ) + + # Update existing data in the task_sample table + task_items = session.execute( + 'SELECT id, config FROM task' + ) + + # Update the task_attachment table + attachments = session.execute( + 'SELECT id, path FROM task_attachment' + ) + + for attachment in attachments: + attachment_id = attachment[0] + attachment_path = attachment[1] + filename = os.path.basename(attachment_path) + url = f"{settings.API_V1_STR}/tasks/attachment/{attachment_path}" + + if filename: + session.execute( + f"UPDATE task_attachment SET filename='{filename}', url='{url}' WHERE id={attachment_id}" + ) + + for task_item in task_items: + task_id = task_item[0] + task_samples = session.execute( + f"SELECT id, task_attachment_ids FROM task_sample WHERE task_id={task_id}" + ) + + for task_sample in task_samples: + task_sample_id = task_sample[0] + attachment_ids = json.loads(task_sample[1]) + # attachment_ids 存储的是字符串[id1, id2, id3]，需要转换成数组 + file_id = attachment_ids[0] + + if not file_id: + continue + + attachment = session.execute( + f"SELECT id, path FROM task_attachment WHERE id={file_id}" + ) + attachment_path = list(attachment)[0][1] + + if attachment_path: + # Update the task_sample table + session.execute( + f"UPDATE task_sample SET file_id={file_id} WHERE id={task_sample_id}" + ) + + + +def downgrade() -> None: + op.drop_table("task_pre_annotation") + with op.batch_alter_table('task_sample') as batch_op: + batch_op.drop_column('file_id') diff --git a/labelu/internal/adapter/persistence/crud_pre_annotation.py b/labelu/internal/adapter/persistence/crud_pre_annotation.py new file mode 100644 index 00000000..71017c6f --- /dev/null +++ b/labelu/internal/adapter/persistence/crud_pre_annotation.py @@ -0,0 +1,104 @@ +from datetime import datetime +import json +from typing import Any, Dict, List, Union + + +from sqlalchemy import case, text +from sqlalchemy.orm import Session +from fastapi.encoders import jsonable_encoder + +from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation +from labelu.internal.adapter.persistence import crud_attachment + + +def batch(db: Session, pre_annotations: List[TaskPreAnnotation]) -> List[TaskPreAnnotation]: + db.bulk_save_objects(pre_annotations, return_defaults=True) + return pre_annotations + + +def list_by( + db: Session, + task_id: Union[int, None], + owner_id: int, + after: Union[int, None], + before: Union[int, None], + pageNo: Union[int, None], + pageSize: int, + sorting: Union[str, None], +) -> List[TaskPreAnnotation]: + + # query filter + query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] + if before: + query_filter.append(TaskPreAnnotation.id < before) + if after: + query_filter.append(TaskPreAnnotation.id > after) + if task_id: + query_filter.append(TaskPreAnnotation.task_id == task_id) + query = db.query(TaskPreAnnotation).filter(*query_filter) + + # default order by id, before need select last items + if before: + query = query.order_by(TaskPreAnnotation.id.desc()) + else: + query = query.order_by(TaskPreAnnotation.id.asc()) + results = ( + query.offset(offset=pageNo * pageSize if pageNo else 0) + .limit(limit=pageSize) + .all() + ) + + # No sorting + + if before: + results.reverse() + + return results + +def list_by_task_id_and_owner_id(db: Session, task_id: int, owner_id: int) -> Dict[str, List[TaskPreAnnotation]]: + pre_annotations = db.query(TaskPreAnnotation).filter( + TaskPreAnnotation.task_id == task_id, + TaskPreAnnotation.deleted_at == None, + TaskPreAnnotation.created_by == owner_id + ).all() + + return pre_annotations + +def get(db: Session, pre_annotation_id: int) -> TaskPreAnnotation: + return ( + db.query(TaskPreAnnotation) + .filter(TaskPreAnnotation.id == pre_annotation_id, TaskPreAnnotation.deleted_at == None) + .first() + ) + + +def get_by_ids(db: Session, pre_annotation_ids: List[int]) -> List[TaskPreAnnotation]: + return ( + db.query(TaskPreAnnotation) + .filter(TaskPreAnnotation.id.in_(pre_annotation_ids), TaskPreAnnotation.deleted_at == None) + .all() + ) + + +def update(db: Session, db_obj: TaskPreAnnotation, obj_in: Dict[str, Any]) -> TaskPreAnnotation: + obj_data = jsonable_encoder(obj_in) + for field in obj_data: + if field in obj_in: + setattr(db_obj, field, obj_in[field]) + db.add(db_obj) + db.flush() + db.refresh(db_obj) + return db_obj + + +def delete(db: Session, pre_annotation_ids: List[int]) -> None: + db.query(TaskPreAnnotation).filter(TaskPreAnnotation.id.in_(pre_annotation_ids)).update( + {TaskPreAnnotation.deleted_at: datetime.now()} + ) + + +def count(db: Session, task_id: int, owner_id: int) -> int: + query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] + if task_id: + query_filter.append(TaskPreAnnotation.task_id == task_id) + return db.query(TaskPreAnnotation).filter(*query_filter).count() diff --git a/labelu/internal/adapter/routers/__init__.py b/labelu/internal/adapter/routers/__init__.py index e7f1b6a5..bb271012 100644 --- a/labelu/internal/adapter/routers/__init__.py +++ b/labelu/internal/adapter/routers/__init__.py @@ -5,6 +5,7 @@ from labelu.internal.adapter.routers import task from labelu.internal.adapter.routers import sample from labelu.internal.adapter.routers import attachment +from labelu.internal.adapter.routers import pre_annotation def add_router(app: FastAPI): @@ -12,3 +13,4 @@ def add_router(app: FastAPI): app.include_router(task.router, prefix=settings.API_V1_STR) app.include_router(attachment.router, prefix=settings.API_V1_STR) app.include_router(sample.router, prefix=settings.API_V1_STR) + app.include_router(pre_annotation.router, prefix=settings.API_V1_STR) diff --git a/labelu/internal/adapter/routers/pre_annotation.py b/labelu/internal/adapter/routers/pre_annotation.py new file mode 100644 index 00000000..9231aaa5 --- /dev/null +++ b/labelu/internal/adapter/routers/pre_annotation.py @@ -0,0 +1,147 @@ +from typing import List, Union + +from sqlalchemy.orm import Session +from fastapi import APIRouter, Depends, Query, status, Security +from fastapi.responses import FileResponse +from fastapi.security import HTTPAuthorizationCredentials + +from labelu.internal.common import db +from labelu.internal.common.security import security +from labelu.internal.common.error_code import ErrorCode +from labelu.internal.common.error_code import LabelUException +from labelu.internal.domain.models.user import User +from labelu.internal.dependencies.user import get_current_user +from labelu.internal.application.service import pre_annotation as service +from labelu.internal.application.command.pre_annotation import CreatePreAnnotationCommand +from labelu.internal.application.command.pre_annotation import DeletePreAnnotationCommand +from labelu.internal.application.response.base import OkResp +from labelu.internal.application.response.base import MetaData +from labelu.internal.application.response.base import CommonDataResp +from labelu.internal.application.response.base import OkRespWithMeta +from labelu.internal.application.response.pre_annotation import PreAnnotationResponse +from labelu.internal.application.response.pre_annotation import CreatePreAnnotationResponse + + +router = APIRouter(prefix="/tasks", tags=["pre_annotations"]) + + +@router.post( + "/{task_id}/pre_annotations", + response_model=OkResp[CreatePreAnnotationResponse], + status_code=status.HTTP_201_CREATED, +) +async def create( + task_id: int, + cmd: List[CreatePreAnnotationCommand], + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Create a pre_annotation. + """ + + # business logic + data = await service.create( + db=db, task_id=task_id, cmd=cmd, current_user=current_user + ) + + # response + return OkResp[CreatePreAnnotationResponse](data=data) + + +@router.get( + "/{task_id}/pre_annotations", + response_model=OkRespWithMeta[List[PreAnnotationResponse]], + status_code=status.HTTP_200_OK, +) +async def list_by( + task_id: int, + sample_name: str = Query(default=None, min_length=1, max_length=255), + after: Union[int, None] = Query(default=None, gt=0), + before: Union[int, None] = Query(default=None, gt=0), + pageNo: Union[int, None] = Query(default=None, ge=0), + pageSize: Union[int, None] = 100, + sort: Union[str, None] = Query( + default=None, regex="(annotated_count|state):(desc|asc)" + ), + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a annotation result. + """ + + if len([i for i in (after, before, pageNo) if i != None]) != 1: + raise LabelUException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + code=ErrorCode.CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR, + ) + + # business logic + data, total = await service.list_by( + db=db, + task_id=task_id, + sample_name=sample_name, + after=after, + before=before, + pageNo=pageNo, + pageSize=pageSize, + sorting=sort, + current_user=current_user, + ) + + # response + meta_data = MetaData(total=total, page=pageNo, size=len(data)) + return OkRespWithMeta[List[PreAnnotationResponse]](meta_data=meta_data, data=data) + + +@router.get( + "/{task_id}/pre_annotations/{pre_annotation_id}", + response_model=OkResp[PreAnnotationResponse], + status_code=status.HTTP_200_OK, +) +async def get( + task_id: int, + pre_annotation_id: int, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a pre annotation result. + """ + + # business logic + data = await service.get( + db=db, task_id=task_id, pre_annotation_id=pre_annotation_id, current_user=current_user + ) + + # response + return OkResp[PreAnnotationResponse](data=data) + + +@router.delete( + "/{task_id}/pre_annotations", + response_model=OkResp[CommonDataResp], + status_code=status.HTTP_200_OK, +) +async def delete( + cmd: DeletePreAnnotationCommand, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + delete a annotation. + """ + + # business logic + data = await service.delete( + db=db, pre_annotation_ids=cmd.pre_annotation_ids, current_user=current_user + ) + + # response + return OkResp[CommonDataResp](data=data) + diff --git a/labelu/internal/application/command/pre_annotation.py b/labelu/internal/application/command/pre_annotation.py new file mode 100644 index 00000000..26c2056e --- /dev/null +++ b/labelu/internal/application/command/pre_annotation.py @@ -0,0 +1,15 @@ +from typing import List +from pydantic import BaseModel, Field + + +class CreatePreAnnotationCommand(BaseModel): + file_id: int = Field( + gt=0, + description="description: attachment file id", + ) + +class DeletePreAnnotationCommand(BaseModel): + pre_annotation_ids: List[int] = Field( + min_items=1, + description="description: pre annotation ids", + ) \ No newline at end of file diff --git a/labelu/internal/application/command/sample.py b/labelu/internal/application/command/sample.py index 83cfac4e..22fa801a 100644 --- a/labelu/internal/application/command/sample.py +++ b/labelu/internal/application/command/sample.py @@ -16,21 +16,19 @@ class ExportType(str, Enum): class CreateSampleCommand(BaseModel): - attachement_ids: List[int] = Field( - min_items=1, + file_id: int = Field( gt=0, description="description: attachment file id", ) data: Union[dict, None] = Field( default=None, - description="description: sample data, include filename, file url, or result", + description="description: annotation result of sample", ) class DeleteSampleCommand(BaseModel): sample_ids: List[int] = Field( min_items=1, - gt=0, description="description: attachment file id", ) @@ -38,7 +36,7 @@ class DeleteSampleCommand(BaseModel): class PatchSampleCommand(BaseModel): data: Union[dict, None] = Field( default=None, - description="description: sample data, include filename, file url, or result", + description="description: sample data, include result", ) annotated_count: Union[int, None] = Field( default=0, description="description: annotate result count" diff --git a/labelu/internal/application/response/attachment.py b/labelu/internal/application/response/attachment.py index 23cfae34..017ee134 100644 --- a/labelu/internal/application/response/attachment.py +++ b/labelu/internal/application/response/attachment.py @@ -9,3 +9,7 @@ class AttachmentResponse(BaseModel): url: Union[str, None] = Field( default=None, description="description: upload file url" ) + filename: Union[str, None] = Field( + default=None, description="description: upload file name" + ) + diff --git a/labelu/internal/application/response/pre_annotation.py b/labelu/internal/application/response/pre_annotation.py new file mode 100644 index 00000000..674bd025 --- /dev/null +++ b/labelu/internal/application/response/pre_annotation.py @@ -0,0 +1,35 @@ +from datetime import datetime + +from typing import List, Union +from pydantic import BaseModel, Field + +from labelu.internal.application.response.base import UserResp + + +class CreatePreAnnotationResponse(BaseModel): + ids: Union[List[int], None] = Field( + default=None, description="description: attachment ids" + ) + + +class PreAnnotationResponse(BaseModel): + id: Union[int, None] = Field(default=None, description="description: annotation id") + file: Union[object, None] = Field( + default=None, description="description: media attachment file" + ) + data: Union[object, None] = Field( + default=None, + description="description: sample data, include filename, file url, or result", + ) + created_at: Union[datetime, None] = Field( + default=None, description="description: task created at time" + ) + created_by: Union[UserResp, None] = Field( + default=None, description="description: task created by" + ) + updated_at: Union[datetime, None] = Field( + default=None, description="description: task updated at time" + ) + updated_by: Union[UserResp, None] = Field( + default=None, description="description: task updated by" + ) diff --git a/labelu/internal/application/response/sample.py b/labelu/internal/application/response/sample.py index ce7e0696..63def954 100644 --- a/labelu/internal/application/response/sample.py +++ b/labelu/internal/application/response/sample.py @@ -19,10 +19,13 @@ class SampleResponse(BaseModel): default=None, description="description: sample file state, NEW is has not start yet, DONE is completed, SKIPPED is skipped", ) - data: Union[dict, None] = Field( + data: Union[object, None] = Field( default=None, description="description: sample data, include filename, file url, or result", ) + file: Union[object, None] = Field( + default=None, description="description: media attachment file" + ) annotated_count: Union[int, None] = Field( default=0, description="description: annotate result count" ) diff --git a/labelu/internal/application/response/task.py b/labelu/internal/application/response/task.py index b3b1e8be..a37be067 100644 --- a/labelu/internal/application/response/task.py +++ b/labelu/internal/application/response/task.py @@ -17,7 +17,7 @@ class TaskResponse(BaseModel): default=None, description="description: task config content" ) media_type: Union[str, None] = Field( - default=None, description="description: task media type: IMAGE, VIDEO" + default=None, description="description: task media type: IMAGE, VIDEO, AUDIO" ) status: Union[str, None] = Field( default=None, diff --git a/labelu/internal/application/service/attachment.py b/labelu/internal/application/service/attachment.py index 1083671e..a9bd4878 100644 --- a/labelu/internal/application/service/attachment.py +++ b/labelu/internal/application/service/attachment.py @@ -101,12 +101,15 @@ async def create( ) attachment_url_path = attachment_relative_path.replace("\\", "/") + attachment_api_url = f"{settings.API_V1_STR}/tasks/attachment/{attachment_url_path}" # add a task file record with db.begin(): attachment = crud_attachment.create( db=db, attachment=TaskAttachment( path=attachment_url_path, + url=attachment_api_url, + filename=filename, created_by=current_user.id, updated_by=current_user.id, task_id=task_id, @@ -116,7 +119,8 @@ async def create( # response return AttachmentResponse( id=attachment.id, - url=f"{settings.API_V1_STR}/tasks/attachment/{attachment_url_path}", + url=attachment_api_url, + filename=filename, ) diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py new file mode 100644 index 00000000..7c7310c3 --- /dev/null +++ b/labelu/internal/application/service/pre_annotation.py @@ -0,0 +1,206 @@ +import json +from datetime import datetime +from typing import Dict, List, Tuple, Optional + +from pathlib import Path +from loguru import logger +from fastapi import status +from sqlalchemy.orm import Session + +from labelu.internal.common.config import settings +from labelu.internal.common.error_code import ErrorCode +from labelu.internal.common.error_code import LabelUException +from labelu.internal.adapter.persistence import crud_task +from labelu.internal.adapter.persistence import crud_pre_annotation +from labelu.internal.adapter.persistence import crud_attachment +from labelu.internal.domain.models.user import User +from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation +from labelu.internal.application.command.pre_annotation import CreatePreAnnotationCommand +from labelu.internal.application.response.base import UserResp +from labelu.internal.application.response.base import CommonDataResp +from labelu.internal.application.response.pre_annotation import CreatePreAnnotationResponse +from labelu.internal.application.response.pre_annotation import PreAnnotationResponse +from labelu.internal.application.response.attachment import AttachmentResponse + +def read_jsonl_file(db: Session, file_id: int) -> List[dict]: + attachment = crud_attachment.get(db, file_id) + if attachment is None: + raise LabelUException(status_code=404, code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND) + + attachment_path = attachment.path + file_full_path = settings.MEDIA_ROOT.joinpath(attachment_path.lstrip("/")) + + try: + with open(file_full_path, "r", encoding="utf-8") as f: + data = f.readlines() + except FileNotFoundError: + raise LabelUException(status_code=404, code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND) + + parsed_data = [json.loads(line) for line in data] + return parsed_data + +async def create( + db: Session, task_id: int, cmd: List[CreatePreAnnotationCommand], current_user: User +) -> CreatePreAnnotationResponse: + with db.begin(): + # check task exist + task = crud_task.get(db=db, task_id=task_id, lock=True) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + def validate_sample_name_exists(file_id: int, pre_annotations_dict: Dict[str, List[TaskPreAnnotation]]) -> List[dict]: + jsonl_content = read_jsonl_file(db, file_id) + + for item in jsonl_content: + sample_name = item.get("sample_name") + pre_annotations = pre_annotations_dict.get(sample_name, []) + + if len(pre_annotations) > 0: + raise LabelUException( + code=ErrorCode.CODE_55002_SAMPLE_NAME_EXISTS, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + # Get all pre_annotations in one query + query_pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id(db=db, task_id=task_id, owner_id=current_user.id) + + pre_annotations_dict = {} + for pre_annotation in query_pre_annotations: + jsonl_content = read_jsonl_file(db, pre_annotation.file_id) + + for item in jsonl_content: + sample_name = item.get("sample_name") + + # 如果字典中已经有这个 sample_name，就添加到列表中 + if sample_name in pre_annotations_dict: + pre_annotations_dict[sample_name].append(pre_annotation) + # 否则，创建一个新的列表 + else: + pre_annotations_dict[sample_name] = [pre_annotation] + + for pre_annotation in cmd: + validate_sample_name_exists(pre_annotation.file_id, pre_annotations_dict) + + pre_annotations = [ + TaskPreAnnotation( + task_id=task_id, + file_id=pre_annotation.file_id, + created_by=current_user.id, + updated_by=current_user.id, + ) + for pre_annotation in cmd + ] + new_pre_annotations = crud_pre_annotation.batch(db=db, pre_annotations=pre_annotations) + + # response + ids = [s.id for s in new_pre_annotations] + return CreatePreAnnotationResponse(ids=ids) + + +async def list_by( + db: Session, + task_id: Optional[int], + sample_name: Optional[str], + after: Optional[int], + before: Optional[int], + pageNo: Optional[int], + pageSize: int, + sorting: Optional[str], + current_user: User, +) -> Tuple[List[PreAnnotationResponse], int]: + + pre_annotations = crud_pre_annotation.list_by( + db=db, + task_id=task_id, + owner_id=current_user.id, + after=after, + before=before, + pageNo=pageNo, + pageSize=pageSize, + sorting=sorting, + ) + + total = crud_pre_annotation.count(db=db, task_id=task_id, owner_id=current_user.id) + + def parse_jsonl_file(file_id: int, sample_name: str) -> List[dict]: + jsonl_content = read_jsonl_file(db, file_id) + # Filter by sample_name + result = [] + for item in jsonl_content: + if sample_name is None or item.get("sample_name") == sample_name: + result.append(item) + return result + + real_sample_name = sample_name[9:] if sample_name else None + parsed_data_list = [parse_jsonl_file(pre_annotation.file_id, real_sample_name) for pre_annotation in pre_annotations] + + def is_contains_sample_name(inputs: List[dict], sample_name: str) -> bool: + return any(data.get("sample_name") == sample_name for data in inputs) + + filtered_pre_annotations = [] + for i, pre_annotation in enumerate(pre_annotations): + if real_sample_name is None or is_contains_sample_name(parsed_data_list[i], real_sample_name): + filtered_pre_annotations.append( + PreAnnotationResponse( + id=pre_annotation.id, + file=AttachmentResponse(id=pre_annotation.file.id, filename=pre_annotation.file.filename, url=pre_annotation.file.url) if pre_annotation.file else None, + created_at=pre_annotation.created_at, + data=parsed_data_list[i], + created_by=UserResp( + id=pre_annotation.owner.id, + username=pre_annotation.owner.username, + ), + updated_at=pre_annotation.updated_at, + updated_by=UserResp( + id=pre_annotation.updater.id, + username=pre_annotation.updater.username, + ), + ) + ) + + return filtered_pre_annotations, total + + +async def get( + db: Session, task_id: int, pre_annotation_id: int, current_user: User +) -> PreAnnotationResponse: + pre_annotation = crud_pre_annotation.get( + db=db, + pre_annotation_id=pre_annotation_id, + ) + + if not pre_annotation: + logger.error("cannot find pre_annotation: {}", pre_annotation_id) + raise LabelUException( + code=ErrorCode.CODE_55001_SAMPLE_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + # response + return PreAnnotationResponse( + id=pre_annotation.id, + file=AttachmentResponse(id=pre_annotation.file.id, filename=pre_annotation.file.filename, url=pre_annotation.file.url) if pre_annotation.file else None, + created_at=pre_annotation.created_at, + created_by=UserResp( + id=pre_annotation.owner.id, + username=pre_annotation.owner.username, + ), + updated_at=pre_annotation.updated_at, + updated_by=UserResp( + id=pre_annotation.updater.id, + username=pre_annotation.updater.username, + ), + ) + +async def delete( + db: Session, pre_annotation_ids: List[int], current_user: User +) -> CommonDataResp: + + with db.begin(): + crud_pre_annotation.delete(db=db, pre_annotation_ids=pre_annotation_ids) + # response + return CommonDataResp(ok=True) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index bdd0411d..65df5195 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -26,7 +26,7 @@ from labelu.internal.application.response.base import CommonDataResp from labelu.internal.application.response.sample import CreateSampleResponse from labelu.internal.application.response.sample import SampleResponse - +from labelu.internal.application.response.attachment import AttachmentResponse async def create( db: Session, task_id: int, cmd: List[CreateSampleCommand], current_user: User @@ -46,7 +46,7 @@ async def create( TaskSample( inner_id=task.last_sample_inner_id + i + 1, task_id=task_id, - task_attachment_ids=str(sample.attachement_ids), + file_id=sample.file_id, created_by=current_user.id, updated_by=current_user.id, data=json.dumps(sample.data, ensure_ascii=False), @@ -96,6 +96,7 @@ async def list_by( state=sample.state, data=json.loads(sample.data), annotated_count=sample.annotated_count, + file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, created_at=sample.created_at, created_by=UserResp( id=sample.owner.id, @@ -132,6 +133,7 @@ async def get( inner_id=sample.inner_id, state=sample.state, data=json.loads(sample.data), + file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, annotated_count=sample.annotated_count, created_at=sample.created_at, created_by=UserResp( @@ -242,7 +244,14 @@ async def export( task = crud_task.get(db=db, task_id=task_id) samples = crud_sample.get_by_ids(db=db, sample_ids=sample_ids) - data = [sample.__dict__ for sample in samples] + data = [] + for sample in samples: + data_dict = sample.__dict__.get('data') + if data_dict is None: + data_dict = {} + file_dict = sample.file.__dict__ if hasattr(sample.file, '__dict__') else {} + data.append({ **sample.__dict__, 'file': file_dict}) + # output data path out_data_dir = Path(settings.MEDIA_ROOT).joinpath( diff --git a/labelu/internal/common/converter.py b/labelu/internal/common/converter.py index fa92ac72..44034ede 100644 --- a/labelu/internal/common/converter.py +++ b/labelu/internal/common/converter.py @@ -57,7 +57,8 @@ def convert_to_json( results = [] for sample in input_data: data = json.loads(sample.get("data")) - + file = sample.get("file", {}) + # change skipped result is invalid annotated_result = json.loads(data.get("result")) if annotated_result and sample.get("state") == "SKIPPED": @@ -73,15 +74,14 @@ def convert_to_json( # 视频文件的标注结果已经保存了 label 键的值，不需要再做转换 if "label" not in tool_result: tool_result["label"] = tool_result.pop("attribute", "") - tool_result["attribute"] = tool_result.pop( - "textAttribute", "" - ) + tool_result.pop("sourceID", None) if tool == "tagTool" or tool == "textTool": tool_result.pop("label") - tool_result.pop("attribute") + if "attribute" in tool_result: + tool_result.pop("attribute") annotations.append(tool_results) @@ -92,14 +92,8 @@ def convert_to_json( { "id": sample.get("id"), "result": annotated_result_str, - "urls": data.get("urls"), - "fileNames": next( - ( - url.split("/")[-1] if url.split("/") else "" - for url in data.get("urls", {}).values() - ), - "", - ), + "url": file.get("url"), + "fileName": file.get("filename"), } ) @@ -133,7 +127,7 @@ def convert_to_coco( # result catetory category_id = 0 logger.info("get categories") - for attr in config.get("attribute", []): + for attr in config.get("attributes", []): category = { "id": category_id, "name": attr.get("value", ""), @@ -163,6 +157,7 @@ def convert_to_coco( # for every annotation media for sample in input_data: annotation_data = json.loads(sample.get("data")) + file = sample.get("file", {}) logger.info("data is: {}", sample) # annotation result @@ -171,18 +166,12 @@ def convert_to_coco( # coco image image = { "id": sample.get("id"), - "fileNames": next( - ( - url.split("/")[-1] if url.split("/") else "" - for url in annotation_data.get("urls", {}).values() - ), - "", - ), + "fileName": file.get("filename"), "width": annotation_result.get("width", 0), "height": annotation_result.get("height", 0), "valid": False if sample.get("state", "") == "SKIPPED" - else annotation_result.get("valid", False), + else annotation_result.get("valid", True), "rotate": annotation_result.get("rotate", 0), } result["images"].append(image) @@ -204,12 +193,12 @@ def convert_to_coco( if tool.get("toolName") == "polygonTool": x_coordinates = [] y_coordinates = [] - for point in tool_result.get("pointList", []): + for point in tool_result.get("points", []): segmentation.append(point.get("x")) segmentation.append(point.get("y")) x_coordinates.append(point.get("x")) y_coordinates.append(point.get("y")) - logger.info("fffffe") + bbox = [ min(x_coordinates), max(y_coordinates), @@ -241,7 +230,7 @@ def convert_to_coco( "area": polygon_area, "bbox": bbox, "category_id": category_name_map_id.get( - tool_result.get("attribute", ""), -1 + tool_result.get("label", ""), -1 ), "order": tool_result.get("order", 0), } @@ -270,13 +259,14 @@ def convert_to_mask( export_files = [] color_list = [] for sample in input_data: + file = sample.get("file", {}) if sample.get("state") != "DONE": continue annotation_data = json.loads(sample.get("data")) logger.info("data is: {}", sample) - filenames = list(annotation_data.get("urls", {}).values()) - if filenames and filenames[0].split("/")[-1]: - file_relative_path_base_name = filenames[0].split("/")[-1].split(".")[0] + filename = file.get("filename") + if filename and filename.split("/")[-1]: + file_relative_path_base_name = filename.split("/")[-1].split(".")[0] else: file_relative_path_base_name = "result" @@ -292,11 +282,11 @@ def convert_to_mask( "result", [] ): polygon = [] - for point in tool_result.get("pointList", []): + for point in tool_result.get("points", []): polygon.append(point.get("x")) polygon.append(point.get("y")) polygons.append(polygon) - polygon_attribute.append(tool_result.get("attribute", "")) + polygon_attribute.append(tool_result.get("label", "")) width = annotation_result.get("width") height = annotation_result.get("height") diff --git a/labelu/internal/common/error_code.py b/labelu/internal/common/error_code.py index 5314a5ae..1505feef 100644 --- a/labelu/internal/common/error_code.py +++ b/labelu/internal/common/error_code.py @@ -82,6 +82,10 @@ class ErrorCode(Enum): TASK_INIT_CODE + 5002, "Sample result format error", ) + CODE_55002_SAMPLE_NAME_EXISTS = ( + TASK_INIT_CODE + 5003, + "Sample name exists", + ) class LabelUException(HTTPException): diff --git a/labelu/internal/domain/models/attachment.py b/labelu/internal/domain/models/attachment.py index 15400f64..a87255a0 100644 --- a/labelu/internal/domain/models/attachment.py +++ b/labelu/internal/domain/models/attachment.py @@ -10,6 +10,8 @@ class TaskAttachment(Base): __tablename__ = "task_attachment" id = Column(Integer, primary_key=True, autoincrement=True, index=True) + filename = Column(String(256), comment="file name") + url = Column(String(256), comment="file url") path = Column(String(256), comment="file storage path") task_id = Column(Integer, ForeignKey("task.id"), index=True) created_by = Column(Integer, ForeignKey("user.id"), index=True) diff --git a/labelu/internal/domain/models/pre_annotation.py b/labelu/internal/domain/models/pre_annotation.py new file mode 100644 index 00000000..c77ddf9a --- /dev/null +++ b/labelu/internal/domain/models/pre_annotation.py @@ -0,0 +1,36 @@ +from datetime import datetime + +from sqlalchemy.schema import Index +from sqlalchemy.orm import relationship +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text + +from labelu.internal.common.db import Base + + +class TaskPreAnnotation(Base): + __tablename__ = "task_pre_annotation" + + id = Column(Integer, primary_key=True, autoincrement=True, index=True) + task_id = Column(Integer, ForeignKey("task.id"), index=True) + file_id = Column(Integer, ForeignKey("task_attachment.id"), index=True) + created_by = Column(Integer, ForeignKey("user.id"), index=True) + updated_by = Column(Integer, ForeignKey("user.id")) + created_at = Column( + DateTime, default=datetime.now, comment="Time a task sample result was created" + ) + updated_at = Column( + DateTime, + default=datetime.now, + onupdate=datetime.now, + comment="Last time a task pre annotation result was updated", + ) + data = Column(Text, comment="task sample pre annotation result") + deleted_at = Column(DateTime, index=True, comment="Task delete time") + + file = relationship("TaskAttachment", foreign_keys=[file_id]) + task = relationship("Task", foreign_keys=[task_id]) + owner = relationship("User", foreign_keys=[created_by]) + updater = relationship("User", foreign_keys=[updated_by]) + + Index("idx_pre_annotation_id_deleted_at", id, deleted_at) + diff --git a/labelu/internal/domain/models/sample.py b/labelu/internal/domain/models/sample.py index 37d7dccd..7004ebdf 100644 --- a/labelu/internal/domain/models/sample.py +++ b/labelu/internal/domain/models/sample.py @@ -24,7 +24,7 @@ class TaskSample(Base): id = Column(Integer, primary_key=True, autoincrement=True, index=True) inner_id = Column(Integer, comment="sample id in a task") task_id = Column(Integer, ForeignKey("task.id"), index=True) - task_attachment_ids = Column(String(255), comment="task sample attachment ids") + file_id = Column(Integer, ForeignKey("task_attachment.id"), index=True) created_by = Column(Integer, ForeignKey("user.id"), index=True) updated_by = Column(Integer, ForeignKey("user.id")) created_at = Column( @@ -49,6 +49,8 @@ class TaskSample(Base): ) deleted_at = Column(DateTime, index=True, comment="Task delete time") + # 由旧的data里的fileNames和urls中的唯一一个，迁移到media中 + file = relationship("TaskAttachment", foreign_keys=[file_id]) task = relationship("Task", foreign_keys=[task_id]) owner = relationship("User", foreign_keys=[created_by]) updater = relationship("User", foreign_keys=[updated_by]) diff --git a/labelu/internal/domain/models/task.py b/labelu/internal/domain/models/task.py index 31a03e50..4ea08541 100644 --- a/labelu/internal/domain/models/task.py +++ b/labelu/internal/domain/models/task.py @@ -43,7 +43,7 @@ class Task(Base): config = Column(Text, comment="task config yaml") media_type = Column( String(32), - comment="task media type: image, video", + comment="task media type: image, video, audio", ) status = Column(String(32), default=TaskStatus.DRAFT.value, comment="task status") created_by = Column(Integer, ForeignKey(column="user.id"), index=True) diff --git a/labelu/main.py b/labelu/main.py index c0e06b72..fa72182e 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -56,7 +56,7 @@ }, { "name": "tasks", - "description": "Task manangement.", + "description": "Task management.", }, { "name": "attachments", @@ -64,7 +64,7 @@ }, { "name": "samples", - "description": "Task sample manangement.", + "description": "Task sample management.", }, ] diff --git a/labelu/tests/data/test.jsonl b/labelu/tests/data/test.jsonl new file mode 100644 index 00000000..645a8b4b --- /dev/null +++ b/labelu/tests/data/test.jsonl @@ -0,0 +1 @@ +{"sample_name":"test.png","config":{"pointTool":[{"color":"#ff6600","key":"标签-1","value":"label-1"}],"polygonTool":[{"color":"#0040ff","key":"标签-1","value":"label-1"}]},"meta_data":{"width":4096,"height":2731,"rotate":0},"annotations":{"pointTool":{"toolName":"pointTool","result":[{"order":1,"id":"a480ea1b-1a49-4449-a426-0388a34ea87b","label":"label-1","x":1250.4922118380061,"y":1301.6993769470405},{"order":2,"id":"f357484c-f911-47fc-af52-fe79b6ba4ec6","label":"label-1","x":969.7694704049844,"y":1805.7242990654206},{"order":3,"id":"485f2bfe-5e89-42aa-ae80-27cfdcb8763f","label":"label-1","x":2335.1028037383176,"y":1512.2414330218069},{"order":4,"id":"7e3cfb5b-d3f7-48d1-bb4d-788977480bf1","label":"label-1","x":2073.520249221184,"y":1780.2040498442368},{"order":5,"id":"cad49f6a-7ef0-4e75-b6b1-7c6796a2d502","label":"label-1","x":2794.467289719626,"y":1837.6246105919004}]},"textTool":{"toolName":"textTool","result":[{"id":"js6htkz785h","type":"text","value":{"text-label-1":"acasdqwe"}}]},"tagTool":{"toolName":"tagTool","result":[{"id":"qw8nbms13h","type":"tag","value":{"tag-label-1":["tag-label-1-1"]}}]}}} \ No newline at end of file diff --git a/labelu/tests/internal/adapter/routers/test_attachment.py b/labelu/tests/internal/adapter/routers/test_attachment.py index acf07b3e..6f9c1361 100644 --- a/labelu/tests/internal/adapter/routers/test_attachment.py +++ b/labelu/tests/internal/adapter/routers/test_attachment.py @@ -28,20 +28,32 @@ def test_upload_file_successful( # run with Path("labelu/tests/data/test.png").open(mode="rb") as f: - new_res = client.post( + png_res = client.post( f"{settings.API_V1_STR}/tasks/{task_id}/attachments", headers=testuser_token_headers, files={"file": ("test.png", f, "image/png")}, ) - # check - json = new_res.json() - assert new_res.status_code == 201 - + json = png_res.json() + assert png_res.status_code == 201 + parts = json["data"]["url"].split("/")[-3:] assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() parts[-1] = parts[-1][:8] + "-test-thumbnail.png" assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() + + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + jsonl_res = client.post( + f"{settings.API_V1_STR}/tasks/{task_id}/attachments", + headers=testuser_token_headers, + files={"file": ("test.jsonl", f, "application/json")}, + ) + + json = jsonl_res.json() + assert jsonl_res.status_code == 201 + + parts = json["data"]["url"].split("/")[-3:] + assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() def test_upload_file_when_task_finished( self, client: TestClient, testuser_token_headers: dict, db: Session diff --git a/labelu/tests/internal/adapter/routers/test_pre_annotation.py b/labelu/tests/internal/adapter/routers/test_pre_annotation.py new file mode 100644 index 00000000..b6900dce --- /dev/null +++ b/labelu/tests/internal/adapter/routers/test_pre_annotation.py @@ -0,0 +1,477 @@ +from sqlalchemy.orm import Session +from pathlib import Path +from fastapi.testclient import TestClient + +from labelu.internal.common.config import settings +from labelu.internal.adapter.persistence import crud_user +from labelu.internal.adapter.persistence import crud_task +from labelu.internal.adapter.persistence import crud_pre_annotation +from labelu.internal.adapter.persistence import crud_sample +from labelu.internal.adapter.persistence import crud_attachment +from labelu.internal.domain.models.task import Task +from labelu.internal.domain.models.attachment import TaskAttachment +from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation +from labelu.internal.domain.models.sample import TaskSample + + +class TestClassTaskPreAnnotationRouter: + def test_create_pre_annotation_successful( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + # prepare data + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + jsonl = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + data = [{"file_id": jsonl.json()["data"]["id"]}] + + # run + r = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + json = r.json() + assert r.status_code == 201 + assert len(json["data"]["ids"]) == 1 + + def test_create_pre_annotation_sample_exists( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + # prepare data + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + res = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + data = [{"file_id": res.json()["data"]["id"]}] + + # run + r = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + res = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + data = [{"file_id": res.json()["data"]["id"]}] + + # run + r = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + assert r.status_code == 400 + + def test_create_sample_task_not_found( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + data = [{"file_id": 1, "data": {}}] + + # run + r = client.post( + f"{settings.API_V1_STR}/tasks/0/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + # check + json = r.json() + assert r.status_code == 404 + assert json["err_code"] == 50002 + + def test_pre_annotation_list_by_page( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + jsonl = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + pre_annotations = [ + TaskPreAnnotation( + task_id=task.id, + file_id=jsonl.json()["data"]["id"], + created_by=current_user.id, + updated_by=current_user.id, + ) + for i in range(14) + ] + + crud_pre_annotation.batch( + db=db, + pre_annotations=pre_annotations, + ) + + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + params={"pageNo": 0, "pageSize": 10}, + ) + + # check + json = r.json() + assert r.status_code == 200 + assert len(json["data"]) == 10 + assert json["data"][0]["id"] == 1 + assert json["meta_data"]["total"] == 14 + + def test_pre_annotation_list_with_sample_name( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + # upload sample file + with Path("labelu/tests/data/test.png").open(mode="rb") as f: + img = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + # upload jsonl file + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + jsonl = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + samples = [ + TaskSample( + task_id=task.id, + file_id=img.json()["data"]["id"], + created_by=current_user.id, + updated_by=current_user.id, + data="{}", + ) + ] + crud_sample.batch( + db=db, + samples=samples, + ) + + pre_annotations = [ + TaskPreAnnotation( + task_id=task.id, + file_id=jsonl.json()["data"]["id"], + created_by=current_user.id, + updated_by=current_user.id, + ) + ] + crud_pre_annotation.batch( + db=db, + pre_annotations=pre_annotations, + ) + + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name={img.json()['data']['filename']}", + headers=testuser_token_headers, + params={"pageNo": 0}, + ) + + # check + json = r.json() + assert r.status_code == 200 + assert json["data"][0]["id"] == 1 + assert json["meta_data"]["total"] == 1 + + def test_pre_annotation_list_with_sample_name_not_found( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + # upload sample file + with Path("labelu/tests/data/test.png").open(mode="rb") as f: + img = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + # upload jsonl file + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: + jsonl = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + samples = [ + TaskSample( + task_id=task.id, + file_id=img.json()["data"]["id"], + created_by=current_user.id, + updated_by=current_user.id, + data="{}", + ) + ] + crud_sample.batch( + db=db, + samples=samples, + ) + + pre_annotations = [ + TaskPreAnnotation( + task_id=task.id, + file_id=jsonl.json()["data"]["id"], + created_by=current_user.id, + updated_by=current_user.id, + ) + ] + crud_pre_annotation.batch( + db=db, + pre_annotations=pre_annotations, + ) + + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name=asdasd.png", + headers=testuser_token_headers, + params={"pageNo": 0}, + ) + + # check + json = r.json() + assert r.status_code == 200 + assert len(json["data"]) == 0 + + def test_pre_annotations_list_by_params_error( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{1}/pre_annotations", + headers=testuser_token_headers, + params={"after": 1, "before": 1, "pageNo": 1, "pageSize": 10}, + ) + + # check + assert r.status_code == 422 + + def test_sample_list_by_sample_name_error( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name=", + headers=testuser_token_headers, + params={"pageNo": 0, "pageSize": 10}, + ) + + # check + assert r.status_code == 422 + + def test_pre_annotation_get( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + pre_annotations = crud_pre_annotation.batch( + db=db, + pre_annotations=[ + TaskPreAnnotation( + task_id=task.id, + file_id=1, + created_by=current_user.id, + updated_by=current_user.id, + ) + ], + ) + + assert pre_annotations[0] is not None + + # run + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations/{pre_annotations[0].id}", + headers=testuser_token_headers, + ) + + # check + assert r.status_code == 200 + + def test_pre_annotation_delete( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + pre_annotations = crud_pre_annotation.batch( + db=db, + pre_annotations=[ + TaskPreAnnotation( + task_id=1, + file_id=1, + created_by=current_user.id, + updated_by=current_user.id, + data="{}", + ) + ], + ) + + # run + data = {"pre_annotation_ids": [pre_annotations[0].id]} + r = client.delete( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + # check + assert r.status_code == 200 + + def test_pre_annotations_delete_not_found( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + + # prepare data + task = crud_task.create( + db=db, + task=Task( + name="name", + description="description", + tips="tips", + created_by=0, + updated_by=0, + ), + ) + + # run + data = {"pre_annotation_ids": [1, 2]} + r = client.delete( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json=data, + ) + + # check + assert r.status_code == 200 diff --git a/labelu/tests/internal/adapter/routers/test_sample.py b/labelu/tests/internal/adapter/routers/test_sample.py index 16dda733..e597fccd 100644 --- a/labelu/tests/internal/adapter/routers/test_sample.py +++ b/labelu/tests/internal/adapter/routers/test_sample.py @@ -25,7 +25,7 @@ def test_create_sample_successful( updated_by=0, ), ) - data = [{"attachement_ids": [1], "data": {}}] + data = [{"file_id": 1, "data": {}}] # run r = client.post( @@ -58,7 +58,7 @@ def test_create_sample_task_status_not_draft( status="CONFIGURED", ), ) - data = [{"attachement_ids": [1], "data": {}}] + data = [{"file_id": 1, "data": {}}] # run r = client.post( @@ -80,7 +80,7 @@ def test_create_sample_task_not_found( ) -> None: # prepare data - data = [{"attachement_ids": [1], "data": {}}] + data = [{"file_id": 1, "data": {}}] # run r = client.post( @@ -115,7 +115,7 @@ def test_sample_list_by_page( samples = [ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=12, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -162,7 +162,7 @@ def test_sample_list_by_before( samples = [ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -210,7 +210,7 @@ def test_sample_list_by_after( samples = [ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -258,7 +258,7 @@ def test_sample_list_with_sort( samples = [ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -352,13 +352,15 @@ def test_sample_get( samples=[ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", ) ], ) + + print(samples) # run r = client.get( @@ -423,7 +425,7 @@ def test_sample_patch( samples=[ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -470,7 +472,7 @@ def test_sample_patch_skip( samples=[ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -560,7 +562,7 @@ def test_sample_delete( samples=[ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, data="{}", @@ -620,7 +622,7 @@ def test_export_sample( name="name", description="description", tips="tips", - config='{"tools":[{"tool":"rectTool","config":{"isShowCursor":false,"showConfirm":false,"skipWhileNoDependencies":false,"drawOutsideTarget":false,"copyBackwardResult":false,"minWidth":1,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"","attributeList":[{"key":"rectTool","value":"rectTool"}]}},{"tool":"pointTool","config":{"upperLimit":10,"isShowCursor":false,"attributeConfigurable":true,"copyBackwardResult":false,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributeList":[{"key":"pointTool","value":"pointTool"}]}},{"tool":"polygonTool","config":{"isShowCursor":false,"lineType":0,"lineColor":0,"drawOutsideTarget":false,"edgeAdsorption":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributeList":[{"key":"polygonTool","value":"polygonTool"}],"lowerLimitPointNum":"4","upperLimitPointNum":100}},{"tool":"lineTool","config":{"isShowCursor":false,"lineType":0,"lineColor":1,"edgeAdsorption":true,"outOfTarget":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"^[\\\\s\\\\S]{1,3}$","lowerLimitPointNum":4,"upperLimitPointNum":"","attributeList":[{"key":"lineTool","value":"lineTool"}]}},{"tool":"tagTool"},{"tool":"textTool"}],"tagList":[{"key":"类别1","value":"class1","isMulti":true,"subSelected":[{"key":"选项1","value":"option1","isDefault":true},{"key":"选项2","value":"option2","isDefault":false}]},{"key":"类别2","value":"class2","isMulti":true,"subSelected":[{"key":"a选项1","value":"aoption1","isDefault":true},{"key":"a选项2","value":"aoption2","isDefault":false}]}],"attribute":[{"key":"RT","value":"RT"}],"textConfig":[{"label":"我的描述","key":"描述的关键","required":true,"default":"","maxLength":200},{"label":"我的描述1","key":"描述的关键1","required":true,"default":"","maxLength":200}],"fileInfo":{"type":"img","list":[{"id":1,"url":"/src/img/example/bear6.webp","result":"[]"}]},"commonAttributeConfigurable":true}', + config='{"tools":[{"tool":"rectTool","config":{"isShowCursor":false,"showConfirm":false,"skipWhileNoDependencies":false,"drawOutsideTarget":false,"copyBackwardResult":false,"minWidth":1,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"","attributes":[{"key":"rectTool","value":"rectTool"}]}},{"tool":"pointTool","config":{"upperLimit":10,"isShowCursor":false,"attributeConfigurable":true,"copyBackwardResult":false,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributes":[{"key":"pointTool","value":"pointTool"}]}},{"tool":"polygonTool","config":{"isShowCursor":false,"lineType":0,"lineColor":0,"drawOutsideTarget":false,"edgeAdsorption":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributes":[{"key":"polygonTool","value":"polygonTool"}],"lowerLimitPointNum":"4","upperLimitPointNum":100}},{"tool":"lineTool","config":{"isShowCursor":false,"lineType":0,"lineColor":1,"edgeAdsorption":true,"outOfTarget":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"^[\\\\s\\\\S]{1,3}$","lowerLimitPointNum":4,"upperLimitPointNum":"","attributes":[{"key":"lineTool","value":"lineTool"}]}},{"tool":"tagTool"},{"tool":"textTool"}],"tagList":[{"key":"类别1","value":"class1","isMulti":true,"subSelected":[{"key":"选项1","value":"option1","isDefault":true},{"key":"选项2","value":"option2","isDefault":false}]},{"key":"类别2","value":"class2","isMulti":true,"subSelected":[{"key":"a选项1","value":"aoption1","isDefault":true},{"key":"a选项2","value":"aoption2","isDefault":false}]}],"attributes":[{"key":"RT","value":"RT"}],"textConfig":[{"label":"我的描述","key":"描述的关键","required":true,"default":"","maxLength":200},{"label":"我的描述1","key":"描述的关键1","required":true,"default":"","maxLength":200}],"fileInfo":{"type":"img","list":[{"id":1,"url":"/src/img/example/bear6.webp","result":"[]"}]},"commonAttributeConfigurable":true}', created_by=0, updated_by=0, ), @@ -630,10 +632,10 @@ def test_export_sample( samples=[ TaskSample( task_id=1, - task_attachment_ids="[1]", + file_id=1, created_by=current_user.id, updated_by=current_user.id, - data='{"result": "{\\"width\\":1256,\\"height\\":647,\\"valid\\":true,\\"rotate\\":0,\\"rectTool\\":{\\"toolName\\":\\"rectTool\\",\\"result\\":[{\\"x\\":76.7636304422194,\\"y\\":86.75077939093666,\\"width\\":156.47058823529457,\\"height\\":86.47058823529437,\\"attribute\\":\\"RT\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"J3eK0yr6\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":1},{\\"x\\":68.52833632457231,\\"y\\":288.5154852732902,\\"width\\":185.29411764705938,\\"height\\":115.29411764705917,\\"attribute\\":\\"rectTool\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"wDIMAnat\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":2}]},\\"pointTool\\":{\\"toolName\\":\\"pointTool\\",\\"result\\":[{\\"x\\":64.41068926574877,\\"y\\":543.8096029203498,\\"isVisible\\":true,\\"attribute\\":\\"无标签\\",\\"valid\\":true,\\"id\\":\\"FRejHry3\\",\\"textAttribute\\":\\"\\",\\"order\\":3},{\\"x\\":257.94010103045525,\\"y\\":552.0448970379969,\\"isVisible\\":true,\\"attribute\\":\\"pointTool\\",\\"valid\\":true,\\"id\\":\\"Hzj1lxHB\\",\\"textAttribute\\":\\"\\",\\"order\\":4}]},\\"polygonTool\\":{\\"toolName\\":\\"polygonTool\\",\\"result\\":[{\\"id\\":\\"YBVRgJYy\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"pointList\\":[{\\"x\\":517.3518657363384,\\"y\\":78.51548527328956},{\\"x\\":472.05774808927936,\\"y\\":247.33901468505476},{\\"x\\":859.1165716186923,\\"y\\":284.3978382144666},{\\"x\\":978.528336324575,\\"y\\":144.39783821446622},{\\"x\\":686.1753951481036,\\"y\\":49.691955861524775}],\\"attribute\\":\\"RT\\",\\"order\\":5},{\\"id\\":\\"miH6P16a\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"pointList\\":[{\\"x\\":550.2930422069267,\\"y\\":399.6919558615258},{\\"x\\":521.4695127951619,\\"y\\":556.1625440968204},{\\"x\\":735.587159853986,\\"y\\":552.0448970379969},{\\"x\\":842.6459833833981,\\"y\\":440.8684264497612},{\\"x\\":764.4106892657508,\\"y\\":395.57430880270226},{\\"x\\":616.1753951481033,\\"y\\":374.9860735085845}],\\"attribute\\":\\"polygonTool\\",\\"order\\":6}]},\\"lineTool\\":{\\"toolName\\":\\"lineTool\\",\\"result\\":[{\\"pointList\\":[{\\"x\\":970.2930422069279,\\"y\\":57.92724997917186,\\"id\\":\\"OIuogYeu\\"},{\\"x\\":1163.8224539716343,\\"y\\":107.33901468505437,\\"id\\":\\"ZpkyExOs\\"},{\\"x\\":1097.9401010304578,\\"y\\":144.39783821446625,\\"id\\":\\"A5pOdHao\\"},{\\"x\\":970.2930422069279,\\"y\\":132.0448970379956,\\"id\\":\\"20Lx7GQZ\\"}],\\"id\\":\\"3U3VY93T\\",\\"valid\\":true,\\"order\\":7,\\"isVisible\\":true,\\"attribute\\":\\"RT\\"},{\\"pointList\\":[{\\"x\\":1040.2930422069282,\\"y\\":284.3978382144667,\\"id\\":\\"1zdkvfsO\\"},{\\"x\\":1229.7048069128111,\\"y\\":284.3978382144667,\\"id\\":\\"L7E6NIDx\\"},{\\"x\\":1213.234218677517,\\"y\\":383.22136762623165,\\"id\\":\\"maDtUPWR\\"},{\\"x\\":1023.822453971634,\\"y\\":403.80960292034933,\\"id\\":\\"xCuDp9g0\\"}],\\"id\\":\\"VvfLXlii\\",\\"valid\\":true,\\"order\\":8,\\"isVisible\\":true,\\"attribute\\":\\"lineTool\\"}]},\\"tagTool\\":{\\"toolName\\":\\"tagTool\\",\\"result\\":[{\\"sourceID\\":\\"\\",\\"id\\":\\"5939weRA\\",\\"result\\":{\\"class1\\":\\"option1\\",\\"class2\\":\\"aoption1\\"}}]},\\"textTool\\":{\\"toolName\\":\\"textTool\\",\\"result\\":[{\\"id\\":\\"OhdGIhFX\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键\\":\\"我的描述\\"}},{\\"id\\":\\"1TN5jPTb\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键1\\":\\"我的描述1\\"}}]}}","urls": {"42": "http://localhost:8000/api/v1/tasks/attachment/upload/6/1/d9c34a05-screen.png"},"fileNames": {"42": ""}}', + data='{"result": "{\\"width\\":1256,\\"height\\":647,\\"valid\\":true,\\"rotate\\":0,\\"rectTool\\":{\\"toolName\\":\\"rectTool\\",\\"result\\":[{\\"x\\":76.7636304422194,\\"y\\":86.75077939093666,\\"width\\":156.47058823529457,\\"height\\":86.47058823529437,\\"label\\":\\"RT\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"J3eK0yr6\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":1},{\\"x\\":68.52833632457231,\\"y\\":288.5154852732902,\\"width\\":185.29411764705938,\\"height\\":115.29411764705917,\\"label\\":\\"rectTool\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"wDIMAnat\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":2}]},\\"pointTool\\":{\\"toolName\\":\\"pointTool\\",\\"result\\":[{\\"x\\":64.41068926574877,\\"y\\":543.8096029203498,\\"isVisible\\":true,\\"label\\":\\"无标签\\",\\"valid\\":true,\\"id\\":\\"FRejHry3\\",\\"textAttribute\\":\\"\\",\\"order\\":3},{\\"x\\":257.94010103045525,\\"y\\":552.0448970379969,\\"isVisible\\":true,\\"label\\":\\"pointTool\\",\\"valid\\":true,\\"id\\":\\"Hzj1lxHB\\",\\"textAttribute\\":\\"\\",\\"order\\":4}]},\\"polygonTool\\":{\\"toolName\\":\\"polygonTool\\",\\"result\\":[{\\"id\\":\\"YBVRgJYy\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"points\\":[{\\"x\\":517.3518657363384,\\"y\\":78.51548527328956},{\\"x\\":472.05774808927936,\\"y\\":247.33901468505476},{\\"x\\":859.1165716186923,\\"y\\":284.3978382144666},{\\"x\\":978.528336324575,\\"y\\":144.39783821446622},{\\"x\\":686.1753951481036,\\"y\\":49.691955861524775}],\\"label\\":\\"RT\\",\\"order\\":5},{\\"id\\":\\"miH6P16a\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"points\\":[{\\"x\\":550.2930422069267,\\"y\\":399.6919558615258},{\\"x\\":521.4695127951619,\\"y\\":556.1625440968204},{\\"x\\":735.587159853986,\\"y\\":552.0448970379969},{\\"x\\":842.6459833833981,\\"y\\":440.8684264497612},{\\"x\\":764.4106892657508,\\"y\\":395.57430880270226},{\\"x\\":616.1753951481033,\\"y\\":374.9860735085845}],\\"label\\":\\"polygonTool\\",\\"order\\":6}]},\\"lineTool\\":{\\"toolName\\":\\"lineTool\\",\\"result\\":[{\\"points\\":[{\\"x\\":970.2930422069279,\\"y\\":57.92724997917186,\\"id\\":\\"OIuogYeu\\"},{\\"x\\":1163.8224539716343,\\"y\\":107.33901468505437,\\"id\\":\\"ZpkyExOs\\"},{\\"x\\":1097.9401010304578,\\"y\\":144.39783821446625,\\"id\\":\\"A5pOdHao\\"},{\\"x\\":970.2930422069279,\\"y\\":132.0448970379956,\\"id\\":\\"20Lx7GQZ\\"}],\\"id\\":\\"3U3VY93T\\",\\"valid\\":true,\\"order\\":7,\\"isVisible\\":true,\\"label\\":\\"RT\\"},{\\"points\\":[{\\"x\\":1040.2930422069282,\\"y\\":284.3978382144667,\\"id\\":\\"1zdkvfsO\\"},{\\"x\\":1229.7048069128111,\\"y\\":284.3978382144667,\\"id\\":\\"L7E6NIDx\\"},{\\"x\\":1213.234218677517,\\"y\\":383.22136762623165,\\"id\\":\\"maDtUPWR\\"},{\\"x\\":1023.822453971634,\\"y\\":403.80960292034933,\\"id\\":\\"xCuDp9g0\\"}],\\"id\\":\\"VvfLXlii\\",\\"valid\\":true,\\"order\\":8,\\"isVisible\\":true,\\"label\\":\\"lineTool\\"}]},\\"tagTool\\":{\\"toolName\\":\\"tagTool\\",\\"result\\":[{\\"sourceID\\":\\"\\",\\"id\\":\\"5939weRA\\",\\"result\\":{\\"class1\\":\\"option1\\",\\"class2\\":\\"aoption1\\"}}]},\\"textTool\\":{\\"toolName\\":\\"textTool\\",\\"result\\":[{\\"id\\":\\"OhdGIhFX\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键\\":\\"我的描述\\"}},{\\"id\\":\\"1TN5jPTb\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键1\\":\\"我的描述1\\"}}]}}","urls": {"42": "http://localhost:8000/api/v1/tasks/attachment/upload/6/1/d9c34a05-screen.png"},"fileNames": {"42": ""}}', annotated_count=1, state="DONE", ) diff --git a/labelu/tests/internal/common/test_converter.py b/labelu/tests/internal/common/test_converter.py index 475dd783..e88bdb4b 100644 --- a/labelu/tests/internal/common/test_converter.py +++ b/labelu/tests/internal/common/test_converter.py @@ -10,7 +10,7 @@ def test_convert_to_json(): { "id": 56, "state": "DONE", - "data": '{"result": "{\\"width\\":1256,\\"height\\":647,\\"valid\\":true,\\"rotate\\":0,\\"rectTool\\":{\\"toolName\\":\\"rectTool\\",\\"result\\":[{\\"x\\":76.7636304422194,\\"y\\":86.75077939093666,\\"width\\":156.47058823529457,\\"height\\":86.47058823529437,\\"attribute\\":\\"RT\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"J3eK0yr6\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":1},{\\"x\\":68.52833632457231,\\"y\\":288.5154852732902,\\"width\\":185.29411764705938,\\"height\\":115.29411764705917,\\"attribute\\":\\"rectTool\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"wDIMAnat\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":2}]},\\"pointTool\\":{\\"toolName\\":\\"pointTool\\",\\"result\\":[{\\"x\\":64.41068926574877,\\"y\\":543.8096029203498,\\"isVisible\\":true,\\"attribute\\":\\"无标签\\",\\"valid\\":true,\\"id\\":\\"FRejHry3\\",\\"textAttribute\\":\\"\\",\\"order\\":3},{\\"x\\":257.94010103045525,\\"y\\":552.0448970379969,\\"isVisible\\":true,\\"attribute\\":\\"pointTool\\",\\"valid\\":true,\\"id\\":\\"Hzj1lxHB\\",\\"textAttribute\\":\\"\\",\\"order\\":4}]},\\"polygonTool\\":{\\"toolName\\":\\"polygonTool\\",\\"result\\":[{\\"id\\":\\"YBVRgJYy\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"pointList\\":[{\\"x\\":517.3518657363384,\\"y\\":78.51548527328956},{\\"x\\":472.05774808927936,\\"y\\":247.33901468505476},{\\"x\\":859.1165716186923,\\"y\\":284.3978382144666},{\\"x\\":978.528336324575,\\"y\\":144.39783821446622},{\\"x\\":686.1753951481036,\\"y\\":49.691955861524775}],\\"attribute\\":\\"RT\\",\\"order\\":5},{\\"id\\":\\"miH6P16a\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"pointList\\":[{\\"x\\":550.2930422069267,\\"y\\":399.6919558615258},{\\"x\\":521.4695127951619,\\"y\\":556.1625440968204},{\\"x\\":735.587159853986,\\"y\\":552.0448970379969},{\\"x\\":842.6459833833981,\\"y\\":440.8684264497612},{\\"x\\":764.4106892657508,\\"y\\":395.57430880270226},{\\"x\\":616.1753951481033,\\"y\\":374.9860735085845}],\\"attribute\\":\\"polygonTool\\",\\"order\\":6}]},\\"lineTool\\":{\\"toolName\\":\\"lineTool\\",\\"result\\":[{\\"pointList\\":[{\\"x\\":970.2930422069279,\\"y\\":57.92724997917186,\\"id\\":\\"OIuogYeu\\"},{\\"x\\":1163.8224539716343,\\"y\\":107.33901468505437,\\"id\\":\\"ZpkyExOs\\"},{\\"x\\":1097.9401010304578,\\"y\\":144.39783821446625,\\"id\\":\\"A5pOdHao\\"},{\\"x\\":970.2930422069279,\\"y\\":132.0448970379956,\\"id\\":\\"20Lx7GQZ\\"}],\\"id\\":\\"3U3VY93T\\",\\"valid\\":true,\\"order\\":7,\\"isVisible\\":true,\\"attribute\\":\\"RT\\"},{\\"pointList\\":[{\\"x\\":1040.2930422069282,\\"y\\":284.3978382144667,\\"id\\":\\"1zdkvfsO\\"},{\\"x\\":1229.7048069128111,\\"y\\":284.3978382144667,\\"id\\":\\"L7E6NIDx\\"},{\\"x\\":1213.234218677517,\\"y\\":383.22136762623165,\\"id\\":\\"maDtUPWR\\"},{\\"x\\":1023.822453971634,\\"y\\":403.80960292034933,\\"id\\":\\"xCuDp9g0\\"}],\\"id\\":\\"VvfLXlii\\",\\"valid\\":true,\\"order\\":8,\\"isVisible\\":true,\\"attribute\\":\\"lineTool\\"}]},\\"tagTool\\":{\\"toolName\\":\\"tagTool\\",\\"result\\":[{\\"sourceID\\":\\"\\",\\"id\\":\\"5939weRA\\",\\"result\\":{\\"class1\\":\\"option1\\",\\"class2\\":\\"aoption1\\"}}]},\\"textTool\\":{\\"toolName\\":\\"textTool\\",\\"result\\":[{\\"id\\":\\"OhdGIhFX\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键\\":\\"我的描述\\"}},{\\"id\\":\\"1TN5jPTb\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键1\\":\\"我的描述1\\"}}]}}","urls": {"42": "http://localhost:8000/api/v1/tasks/attachment/upload/6/1/d9c34a05-screen.png"},"fileNames": {"42": ""}}', + "data": '{"result": "{\\"width\\":1256,\\"height\\":647,\\"valid\\":true,\\"rotate\\":0,\\"rectTool\\":{\\"toolName\\":\\"rectTool\\",\\"result\\":[{\\"x\\":76.7636304422194,\\"y\\":86.75077939093666,\\"width\\":156.47058823529457,\\"height\\":86.47058823529437,\\"label\\":\\"RT\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"J3eK0yr6\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":1},{\\"x\\":68.52833632457231,\\"y\\":288.5154852732902,\\"width\\":185.29411764705938,\\"height\\":115.29411764705917,\\"label\\":\\"rectTool\\",\\"valid\\":true,\\"isVisible\\":true,\\"id\\":\\"wDIMAnat\\",\\"sourceID\\":\\"\\",\\"textAttribute\\":\\"\\",\\"order\\":2}]},\\"pointTool\\":{\\"toolName\\":\\"pointTool\\",\\"result\\":[{\\"x\\":64.41068926574877,\\"y\\":543.8096029203498,\\"isVisible\\":true,\\"label\\":\\"无标签\\",\\"valid\\":true,\\"id\\":\\"FRejHry3\\",\\"textAttribute\\":\\"\\",\\"order\\":3},{\\"x\\":257.94010103045525,\\"y\\":552.0448970379969,\\"isVisible\\":true,\\"label\\":\\"pointTool\\",\\"valid\\":true,\\"id\\":\\"Hzj1lxHB\\",\\"textAttribute\\":\\"\\",\\"order\\":4}]},\\"polygonTool\\":{\\"toolName\\":\\"polygonTool\\",\\"result\\":[{\\"id\\":\\"YBVRgJYy\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"points\\":[{\\"x\\":517.3518657363384,\\"y\\":78.51548527328956},{\\"x\\":472.05774808927936,\\"y\\":247.33901468505476},{\\"x\\":859.1165716186923,\\"y\\":284.3978382144666},{\\"x\\":978.528336324575,\\"y\\":144.39783821446622},{\\"x\\":686.1753951481036,\\"y\\":49.691955861524775}],\\"label\\":\\"RT\\",\\"order\\":5},{\\"id\\":\\"miH6P16a\\",\\"valid\\":true,\\"isVisible\\":true,\\"textAttribute\\":\\"\\",\\"points\\":[{\\"x\\":550.2930422069267,\\"y\\":399.6919558615258},{\\"x\\":521.4695127951619,\\"y\\":556.1625440968204},{\\"x\\":735.587159853986,\\"y\\":552.0448970379969},{\\"x\\":842.6459833833981,\\"y\\":440.8684264497612},{\\"x\\":764.4106892657508,\\"y\\":395.57430880270226},{\\"x\\":616.1753951481033,\\"y\\":374.9860735085845}],\\"label\\":\\"polygonTool\\",\\"order\\":6}]},\\"lineTool\\":{\\"toolName\\":\\"lineTool\\",\\"result\\":[{\\"points\\":[{\\"x\\":970.2930422069279,\\"y\\":57.92724997917186,\\"id\\":\\"OIuogYeu\\"},{\\"x\\":1163.8224539716343,\\"y\\":107.33901468505437,\\"id\\":\\"ZpkyExOs\\"},{\\"x\\":1097.9401010304578,\\"y\\":144.39783821446625,\\"id\\":\\"A5pOdHao\\"},{\\"x\\":970.2930422069279,\\"y\\":132.0448970379956,\\"id\\":\\"20Lx7GQZ\\"}],\\"id\\":\\"3U3VY93T\\",\\"valid\\":true,\\"order\\":7,\\"isVisible\\":true,\\"label\\":\\"RT\\"},{\\"points\\":[{\\"x\\":1040.2930422069282,\\"y\\":284.3978382144667,\\"id\\":\\"1zdkvfsO\\"},{\\"x\\":1229.7048069128111,\\"y\\":284.3978382144667,\\"id\\":\\"L7E6NIDx\\"},{\\"x\\":1213.234218677517,\\"y\\":383.22136762623165,\\"id\\":\\"maDtUPWR\\"},{\\"x\\":1023.822453971634,\\"y\\":403.80960292034933,\\"id\\":\\"xCuDp9g0\\"}],\\"id\\":\\"VvfLXlii\\",\\"valid\\":true,\\"order\\":8,\\"isVisible\\":true,\\"label\\":\\"lineTool\\"}]},\\"tagTool\\":{\\"toolName\\":\\"tagTool\\",\\"result\\":[{\\"sourceID\\":\\"\\",\\"id\\":\\"5939weRA\\",\\"result\\":{\\"class1\\":\\"option1\\",\\"class2\\":\\"aoption1\\"}}]},\\"textTool\\":{\\"toolName\\":\\"textTool\\",\\"result\\":[{\\"id\\":\\"OhdGIhFX\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键\\":\\"我的描述\\"}},{\\"id\\":\\"1TN5jPTb\\",\\"sourceID\\":\\"\\",\\"value\\":{\\"描述的关键1\\":\\"我的描述1\\"}}]}}","urls": {"42": "http://localhost:8000/api/v1/tasks/attachment/upload/6/1/d9c34a05-screen.png"},"fileNames": {"42": ""}}', "annotated_count": 4, } ] @@ -110,7 +110,7 @@ def test_convert_to_json(): ], }, ], - "attribute": [{"key": "RT", "value": "RT"}], + "attributes": [{"key": "RT", "value": "RT"}], "textConfig": [ { "label": "我的描述", diff --git a/pyproject.toml b/pyproject.toml index 24daf43f..db08af9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '0.7.562' +version = '1.0.0-alpha.67' description = "" license = "Apache-2.0" authors = ["pengjinhu "] @@ -29,19 +29,16 @@ pillow = "^9.3.0" alembic = "^1.9.4" - [[tool.poetry.source]] name = "testpypi" url = "https://test.pypi.org/legacy/" default = false secondary = false - [tool.poetry.group.dev.dependencies] black = "^22.10.0" flake8 = "^5.0.4" - [tool.poetry.group.test.dependencies] pytest = "^7.2.0" pytest-cov = "^4.0.0" diff --git a/scripts/resolve_frontend.sh b/scripts/resolve_frontend.sh index 2b534299..c26fa9dd 100644 --- a/scripts/resolve_frontend.sh +++ b/scripts/resolve_frontend.sh @@ -53,17 +53,20 @@ filename=$(basename $url) echo "final url: $url" echo "filename: $filename" -# 下载zip文件 -wget $url +# 如果第一个参数为true，则下载zip文件 +if [ "$1" = "true" ]; then + # 下载zip文件 + wget $url -# 解压zip文件 -unzip -o $filename + # 解压zip文件 + unzip -o $filename -# 删除下载的zip文件 -rm $filename + # 删除下载的zip文件 + rm $filename -# 移动到指定目录 -mv dist/* labelu/internal/statics + # 移动到指定目录 + mv dist/* labelu/internal/statics -# 删除空目录 -rm -rf dist \ No newline at end of file + # 删除空目录 + rm -rf dist +fi \ No newline at end of file