diff --git a/.github/workflows/2.x-aarch64.yml b/.github/workflows/2.x-aarch64.yml index eaae6533..6aa9feca 100644 --- a/.github/workflows/2.x-aarch64.yml +++ b/.github/workflows/2.x-aarch64.yml @@ -140,7 +140,7 @@ jobs: distro: ubuntu22.04 run: | apt update -y > /dev/null - apt install git cmake build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config -y > /dev/null + apt install git cmake build-essential libexpat1-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config -y > /dev/null cd TDengine mkdir debug ||: diff --git a/.github/workflows/3.0-alpine.yml b/.github/workflows/3.0-alpine.yml deleted file mode 100644 index 77101f60..00000000 --- a/.github/workflows/3.0-alpine.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Alpine (3.0 alpine) - -on: - schedule: - - cron: "10 16 * * *" - push: - branches: - - develop - - 3.0 - - main - pull_request: - branches: - - develop - - 3.0 - - main - -env: - TOOLS_BUILD_TYPE: Release - PR_NUMBER: ${{ github.event.number }} - -jobs: - check-changed: - runs-on: ubuntu-latest - outputs: - changedflag: ${{ steps.changedflag.outputs.changedflag }} - steps: - - name: Step that prints name of pull request's base branch - run: | - echo "Pull request's base branch is: ${BASE_BRANCH}" - echo "Pull request's branch is: ${GITHUB_REF##*/}" - echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" - env: - BASE_BRANCH: ${{ github.base_ref }} - if: github.event_name == 'pull_request' - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.2 - - - name: List all changed files - run: | - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - echo "$file was changed" - done - - - name: Get specific changed files - id: changed-files-specific - uses: tj-actions/changed-files@v23.2 - with: - files: | - src/* - inc/* - deps/CMakeLists.txt - .github/workflows/3.0-alpine.yml - - - name: Run step if any of the listed files above change - id: changedflag - if: steps.changed-files-specific.outputs.any_changed == 'true' - run: | - echo "One or more files listed above has changed." > ~/changed.log - echo "::set-output name=changedflag::true" - - build: - runs-on: ubuntu-latest - needs: check-changed - - steps: - - name: Step that prints name of pull request's base branch - run: | - echo ${{needs.check-changed.outputs.changedflag}} - echo "Pull request's base branch is: ${BASE_BRANCH}" - echo "Pull request's branch is: ${GITHUB_REF##*/}" - echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" - env: - BASE_BRANCH: ${{ github.base_ref }} - if: github.event_name == 'pull_request' - - - name: setup Alpine - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: jirutka/setup-alpine@v1 - - - - name: Run script inside Alpine chroot as root - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - run: | - cat /etc/alpine-release - apk add argp-standalone bash curl cmake gcc g++ git go procps lsof make valgrind linux-headers libunwind libunwind-dev tzdata wget jansson-dev snappy-dev xz-dev zlib-dev - shell: alpine.sh --root {0} - - - name: Build & Install TDengine - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - shell: alpine.sh --root {0} - run: | - git clone --branch ${{ github.event.pull_request.base.ref }} --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DBUILD_HTTP=false && make -j2 && make install - if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taos ]] - then - echo "TDengien build failure" - exit 1 - fi - - if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] - then - echo "taos-tools build failure" - exit 1 - fi - diff --git a/.github/workflows/3.0-archlinux-release.yml b/.github/workflows/3.0-archlinux-release.yml deleted file mode 100644 index 9b31a57e..00000000 --- a/.github/workflows/3.0-archlinux-release.yml +++ /dev/null @@ -1,206 +0,0 @@ -name: Arch Linux (3.0 release build) - -on: - schedule: - - cron: "10 16 * * *" - push: - branches: - - develop - - 3.0 - - main - pull_request: - branches: - - develop - - 3.0 - - main - -env: - TOOLS_BUILD_TYPE: Release - PR_NUMBER: ${{ github.event.number }} - -jobs: - check-changed: - runs-on: ubuntu-latest - outputs: - output1: ${{ steps.step1.outputs.test }} - output2: ${{ steps.step2.outputs.test }} - changedflag: ${{ steps.changedflag.outputs.changedflag }} - steps: - - name: Step that prints name of pull request's base branch - run: | - echo "Pull request's base branch is: ${BASE_BRANCH}" - echo "Pull request's branch is: ${GITHUB_REF##*/}" - echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" - env: - BASE_BRANCH: ${{ github.base_ref }} - if: github.event_name == 'pull_request' - - - name: checkout taos-tools - uses: actions/checkout@v3 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.2 - - - name: List all changed files - run: | - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - echo "$file was changed" - done - - - name: Get specific changed files - id: changed-files-specific - uses: tj-actions/changed-files@v23.2 - with: - files: | - src/* - inc/* - deps/CMakeLists.txt - .github/workflows/3.0-archlinux-release.yml - - - name: Run step if any of the listed files above change - id: changedflag - if: steps.changed-files-specific.outputs.any_changed == 'true' - run: | - echo "One or more files listed above has changed." > ~/changed.log - echo "::set-output name=changedflag::true" - - build: - runs-on: ubuntu-latest - needs: check-changed - container: docker.io/archlinux:latest - - steps: - - name: Step that prints name of pull request's base branch - run: | - echo ${{needs.check-changed.outputs.changedflag}} - echo "Pull request's base branch is: ${BASE_BRANCH}" - echo "Pull request's branch is: ${GITHUB_REF##*/}" - echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" - env: - BASE_BRANCH: ${{ github.base_ref }} - if: github.event_name == 'pull_request' - - - name: install packages for build - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - run: | - pacman -Syu --noconfirm > /dev/null - pacman -Sy git --noconfirm > /dev/null - - - name: Checkout TDengine - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: actions/checkout@v2 - with: - submodules: recursive - repository: 'taosdata/TDengine' - path: 'TDengine' - ref: ${{ github.event.pull_request.base.ref }} - - - name: Change time zone - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - run: | - echo "disable timezone changing" - #timedatectl set-timezone Asia/Shanghai - #date - - - name: Set up Go - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: actions/setup-go@v3 - with: - go-version: 1.17 - - - name: Set up Rust - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - uses: actions/cache@v3 - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - id: cache-rust - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ steps.setup-rust.outputs.rustc_hash }} - - - name: first build TDengine 3.0 - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - run: | - pacman -Sy --noconfirm gcc make cmake gcc-libs gflags pkg-config python3 python-pip snappy zlib > /dev/null - gcc --version - python3 -m pip install --upgrade pip > /dev/null 2>&1 - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true && make -j2 > /dev/null && make install > /dev/null - if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] - then - echo "TDengine build failure" - exit 1 - fi - - - name: checkout taos-tools - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: actions/checkout@v3 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - - name: Checkout taos-tools to PR number - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - run: | - git config --global --add safe.directory /__w/taos-tools/taos-tools - git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge - git checkout -qf FETCH_HEAD - - - name: build taos-tools - if: | - (needs.check-changed.outputs.changedflag == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - run: | - mkdir debug ||: - cd debug - cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DWEBSOCKET=true > /dev/null - make -j8 > /dev/null && make install > /dev/null - if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] - then - echo "taos-tools build failure" - exit 1 - fi diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml deleted file mode 100644 index 6ba0881a..00000000 --- a/.github/workflows/3.0-non-x64.yml +++ /dev/null @@ -1,193 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Ubuntu (3.x non-x64 build) - -# Controls when the workflow will run -on: - schedule: - - cron: "10 16 * * *" - # Triggers the workflow on push or pull request events but only for the develop branch - push: - branches: - - develop - - 3.0 - - main - pull_request: - branches: - - develop - - 3.0 - - main - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - TOOLS_BUILD_TYPE: Debug - PR_NUMBER: ${{ github.event.number }} - -jobs: - build_job: - runs-on: ubuntu-20.04 - name: Build on ${{ matrix.distro }} ${{ matrix.arch }} - - # Run steps on a matrix of 3 arch/distro combinations - strategy: - matrix: - include: - - arch: aarch64 - distro: ubuntu20.04 - # - arch: armv7 - # distro: ubuntu20.04 - - steps: - - uses: actions/checkout@v3 - - uses: uraimo/run-on-arch-action@v2.1.1 - name: Run setup - id: setup - with: - arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} - ref: 'develop' - path: 'taos-tools' - # Set an output parameter `uname` for use in subsequent steps - run: | - uname -a - echo ::set-output name=uname::$(uname -a) - - - name: Step that prints name of pull request's base branch - run: | - echo "The uname output was ${{ steps.setup.outputs.uname }}" - echo "Pull request's base branch is: ${BASE_BRANCH}" - echo "Pull request's branch is: ${GITHUB_REF##*/}" - echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" - env: - BASE_BRANCH: ${{ github.base_ref }} - if: github.event_name == 'pull_request' - || github.event_name == 'schedule' - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - submodules: false - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.2 - - - name: List all changed files - run: | - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - echo "$file was changed" - done - - - name: Get specific changed files - id: changed-files-specific - uses: tj-actions/changed-files@v23.2 - with: - files: | - src/* - inc/* - deps/CMakeLists.txt - packaging/tools/* - .github/workflows/3.0-non-x64.yml - - - name: Run step if any of the listed files above change - if: steps.changed-files-specific.outputs.any_changed == 'true' - run: | - echo "One or more files listed above has changed." - - - name: Change time zone - if: steps.changed-files-specific.outputs.any_changed == 'true' - run: | - echo "The uname output was ${{ steps.setup.outputs.uname }}" - sudo timedatectl set-timezone Asia/Shanghai - sudo date - - # - name: Build TDengine and taos-tools on armv7 - # if: | - # (steps.changed-files-specific.outputs.any_changed == 'true' - # && github.event_name == 'pull_request') - # || github.event_name == 'push' - # uses: uraimo/run-on-arch-action@v2.1.1 - # with: - # arch: armv7 - # distro: ubuntu20.04 - # run: | - # echo "The uname output was ${{ steps.setup.outputs.uname }}" - # apt update -y > /dev/null - # apt install git cmake build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config -y > /dev/null - # cd TDengine - # mkdir armv7-debug ||: - # cd armv7-debug - # cmake .. -DBUILD_TOOLS=true > /dev/null - # make > /dev/null && make install > /dev/null - # if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] - # then - # echo "TDengine build failure" - # exit 1 - # fi - - - name: Build TDengine 3.0 and taos-tools and run test on aarch64 - if: | - (steps.changed-files-specific.outputs.any_changed == 'true' - && github.event_name == 'pull_request') - || github.event_name == 'push' - || github.event_name == 'schedule' - uses: uraimo/run-on-arch-action@v2.1.1 - with: - arch: aarch64 - distro: ubuntu20.04 - run: | - echo "Install packages on ${{ steps.setup.outputs.uname }}" - apt update -y > /dev/null - apt install -y cmake build-essential git libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev > /dev/null - apt install libgflags2.2 libgflags-dev -y > /dev/null - - echo "clone TDengine ${{ github.event.pull_request.base.ref }} on ${{ steps.setup.outputs.uname }}" - - git clone --branch ${{ github.event.pull_request.base.ref }} --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 - - echo "build TDengine 3.0 on ${{ steps.setup.outputs.uname }}" - cd TDengine || exit 1 - mkdir debug ||: - cd debug || exit 1 - cmake .. > /dev/null && make -j2 > /dev/null && make install > /dev/null - if [[ ! -f /usr/local/taos/bin/taosd ]] - then - echo "TDengine build failure" - exit 1 - fi - - echo "Checkout taos-tools to PR number on ${{ steps.setup.outputs.uname }}" - git config --global --add safe.directory /home/runner/work/taos-tools/taos-tools - cd /home/runner/work/taos-tools/taos-tools - - if [ ! -z "${{env.PR_NUMBER}}" ]; then - git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge - git checkout -qf FETCH_HEAD - fi - - mkdir debug ||: - cd debug - cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} > /dev/null && make -j2 > /dev/null && make install > /dev/null - if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] - then - echo "taos-tools build failure" - exit 1 - fi - - echo "TEST: old cases" - cd ../tests - - apt install python3 python3-pip -y > /dev/null - pip3 install --upgrade pip > /dev/null - pip3 install decorator numpy psutil pandas faker toml > /dev/null - pip3 install taospy > /dev/null - pip3 install fabric2 > /dev/null - - python3 ./test.py -f taosdump/old/taosdumpTest.py && echo -e "\033[32m taosdumpTest.py success! \033[0m" || echo -e "\033[31m taosdumpTest.py failed! \033[0m" | tee -a ~/taosdump-failed.txt - - if [ -f ~/taosdump-failed.txt ]; then - cat ~/taosdump-failed.txt; - exit 1; - fi diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml deleted file mode 100644 index 2a5c3248..00000000 --- a/.github/workflows/commitlint.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Lint Commit Messages -on: [pull_request, push] - -jobs: - commitlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: wagoid/commitlint-github-action@v4 diff --git a/VERSION b/VERSION index 2d9d10b2..8ea9aea8 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ -taosbenchmark-3.2.3 -taosdump-2.5.4 -2.5.4 +taosbenchmark-3.3.1 +taosdump-2.6.0 +2.6.0 \ No newline at end of file diff --git a/case/exportCsv.json b/case/exportCsv.json new file mode 100644 index 00000000..f031266c --- /dev/null +++ b/case/exportCsv.json @@ -0,0 +1,77 @@ +{ + "filetype": "csvfile", + "csvPath": "/root/csv/", + "num_of_records_per_req": 10000, + "databases": [ + { + "dbinfo": { + "name": "csvdb" + }, + "super_tables": [ + { + "name": "batchTable", + "childtable_count": 10, + "insert_rows": 50, + "childtable_prefix": "d", + "timestamp_step": 10, + "start_timestamp":1600000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + }, + { + "name": "interlaceTable", + "childtable_count": 5, + "insert_rows": 100, + "interlace_rows": 10, + "childtable_prefix": "e", + "timestamp_step": 1000, + "start_timestamp":1700000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min":16}, + { "type": "double", "name": "dc", "min":16}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/case/insertBindVGroup.json b/case/insertBindVGroup.json new file mode 100644 index 00000000..beaadfb5 --- /dev/null +++ b/case/insertBindVGroup.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 200, + "thread_count": 20, + "thread_bind_vgroup": "yes", + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "binddb", + "drop": "yes", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 100, + "interlace_rows": 10, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":1500000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/case/insertCompress.json b/case/insertCompress.json new file mode 100644 index 00000000..6e4d9886 --- /dev/null +++ b/case/insertCompress.json @@ -0,0 +1,67 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 3, + "create_table_thread_count": 10, + "prepare_rand": 1000, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "vgroups": 3, + "replica": 1, + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 10, + "start_timestamp":1600000000000, + "disorder_ratio": 10, + "update_ratio": 5, + "delete_ratio": 1, + "disorder_fill_interval": 300, + "update_fill_interval": 25, + "generate_row_rule": 2, + "columns": [ + { "type": "bool", "name": "bc", "encode":"bit-packing", "compress":"zstd", "level":"high" }, + { "type": "float", "name": "fc", "encode":"delta-d", "compress":"zlib", "level":"medium", "max": 100, "min": 0 }, + { "type": "double", "name": "dc", "encode":"delta-d", "compress":"xz", "level":"low", "max": 100, "min": 0 }, + { "type": "tinyint", "name": "ti", "encode":"delta-i", "compress":"zstd", "level":"high", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0, "compress":"zlib" }, + { "type": "int", "name": "ic", "max": 100, "min": 0, "compress":"zstd" }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0, "encode":"delta-i" }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0, "level":"high" }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0, "level":"medium" }, + { "type": "uint", "name": "ui", "max": 100, "min": 0, "level":"low" }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0, "compress":"xz", "level":"medium" }, + { "type": "binary", "name": "bin", "len": 32, "compress":"zstd"}, + { "type": "nchar", "name": "nch", "len": 64, "compress":"xz"} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/case/insertPrimaryKey.json b/case/insertPrimaryKey.json new file mode 100644 index 00000000..572a31e7 --- /dev/null +++ b/case/insertPrimaryKey.json @@ -0,0 +1,75 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 1000, + "create_table_thread_count": 2, + "thread_count": 10, + "prepared_rand": 1000, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "primarydb", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 10, + "start_timestamp":1500000000000, + "primary_key":1, + "repeat_ts_min": 10, + "repeat_ts_max": 10, + "disorder_ratio":0, + "update_ratio": 0, + "delete_ratio": 0, + "disorder_fill_interval": 100, + "update_fill_interval": 25, + "generate_row_rule": 2, + "columns": [ + { "type": "int", "name": "pk", "max": 100000, "min": 0, "gen":"order","fillNull":"false"}, + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1, "gen":"order"}, + { "type": "double", "name": "dc", "min":10, "gen":"order"}, + { "type": "tinyint", "name": "ti", "gen":"order"}, + { "type": "smallint", "name": "si", "gen":"order"}, + { "type": "int", "name": "ic", "gen":"order", "fillNull":"false"}, + { "type": "bigint", "name": "bi", "gen":"order"}, + { "type": "utinyint", "name": "uti", "gen":"order"}, + { "type": "usmallint", "name": "usi", "gen":"order"}, + { "type": "uint", "name": "ui", "gen":"order"}, + { "type": "ubigint", "name": "ubi", "gen":"order"}, + { "type": "binary", "name": "bin", "len": 32, "gen":"order"}, + { "type": "nchar", "name": "nch", "len": 64, "gen":"order"} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ], + "sqls" : [ + "create tmsa tmsa1 on test.meters ...", + "create recursive tsma tmsa2 on test.tmsa1 ..." + ] + } + ] + } + ] +} diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 6f135fe3..84492fa2 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -96,6 +96,7 @@ ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") COMMAND cmake -E copy ${PROJECT_SOURCE_DIR}/deps/jansson-value.c ${PROJECT_SOURCE_DIR}/deps/jansson/src/value.c ) + MESSAGE(STATUS "deps' cmake detected BUILD_TYPE: ${CMAKE_BUILD_TYPE}") ExternalProject_Add( deps-libargp PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libargp/ @@ -114,7 +115,7 @@ ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") deps-snappy PREFIX ${CMAKE_CURRENT_BINARY_DIR}/snappy SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/snappy - CONFIGURE_COMMAND cmake -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_SHARED_LIBS=OFF -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -S ${PROJECT_SOURCE_DIR}/deps/snappy + CONFIGURE_COMMAND cmake -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_SHARED_LIBS=OFF -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -S ${PROJECT_SOURCE_DIR}/deps/snappy ) IF (${TOOLS_BUILD_TYPE} MATCHES "Debug") diff --git a/example/dmeters.json b/example/dmeters.json new file mode 100644 index 00000000..8840f5b3 --- /dev/null +++ b/example/dmeters.json @@ -0,0 +1,79 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 5, + "create_table_thread_count": 8, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 2000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "keep_trying": 10000, + "trying_interval": 10000, + "databases": [ + { + "dbinfo": { + "name": "dmeters", + "drop": "no", + "vgroups": 4, + "duration": "5d", + "keep": "30d", + "pages": 512, + "wal_retention_period": 1, + "wal_retention_size": 10, + "cachemodel": "'both'", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 99999999999999, + "interlace_rows": 1, + "insert_interval": 10000, + "start_timestamp": "now", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "4*sin(x)+10*random(5)+10"}, + { "type": "INT", "name": "voltage", "fun": "1*square(0,60,50,0)+100*random(20)+120"}, + { "type": "FLOAT", "name": "phase", "fun": "1*saw(0,40,40,0)+50*random(10)+30"} + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/example/filldmeters.json b/example/filldmeters.json new file mode 100644 index 00000000..7cc74482 --- /dev/null +++ b/example/filldmeters.json @@ -0,0 +1,81 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 5, + "create_table_thread_count": 8, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 2000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "keep_trying": 10000, + "trying_interval": 10000, + "databases": [ + { + "dbinfo": { + "name": "dmeters", + "drop": "no", + "vgroups": 4, + "duration": "5d", + "keep": "30d", + "pages": 512, + "wal_retention_period": 1, + "wal_retention_size": 10, + "cachemodel": "'both'", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "yes", + "childtable_count": 100000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 99999999999999, + "interlace_rows": 1, + "insert_interval": 10000, + "start_timestamp": "now", + "start_fillback_time": "auto", + "timestamp_step": 10000, + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "4*sin(x)+10*random(5)+10"}, + { "type": "INT", "name": "voltage", "fun": "1*square(0,60,50,0)+100*random(20)+120"}, + { "type": "FLOAT", "name": "phase", "fun": "1*saw(0,40,40,0)+50*random(10)+30"} + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/example/insert.json b/example/insert.json index 17491611..1e40883b 100644 --- a/example/insert.json +++ b/example/insert.json @@ -7,27 +7,27 @@ "password": "taosdata", "connection_pool_size": 8, "thread_count": 4, - "create_table_thread_count": 7, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", - "insert_interval": 0, - "interlace_rows": 100, - "num_of_records_per_req": 100, + "num_of_records_per_req": 10000, "prepared_rand": 10000, "chinese": "no", "escape_character": "yes", + "continue_if_fail": "no", "databases": [ { "dbinfo": { "name": "test", "drop": "yes", + "vgroups": 4, "precision": "ms" }, "super_tables": [ { "name": "meters", "child_table_exists": "no", - "childtable_count": 10000, + "childtable_count": 1000, "childtable_prefix": "d", "auto_create_table": "no", "batch_create_tbl_num": 5, @@ -36,13 +36,11 @@ "non_stop_mode": "no", "line_protocol": "line", "insert_rows": 10000, - "childtable_limit": 10, - "childtable_offset": 100, + "childtable_limit": 0, + "childtable_offset": 0, "interlace_rows": 0, "insert_interval": 0, "partial_col_num": 0, - "disorder_ratio": 0, - "disorder_range": 1000, "timestamp_step": 10, "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", @@ -50,27 +48,13 @@ "use_sample_ts": "no", "tags_file": "", "columns": [ - { - "type": "FLOAT", - "name": "current", - "count": 1, - "max": 12, - "min": 8 - }, + {"type": "FLOAT", "name": "current", "count": 1, "max": 12, "min": 8 }, { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } ], "tags": [ - { - "type": "TINYINT", - "name": "groupid", - "max": 10, - "min": 1 - }, - { - "name": "location", - "type": "BINARY", - "len": 16, + {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}, + {"type": "BINARY", "name": "location", "len": 16, "values": ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View", "Sunnyvale", "Santa Clara", "Cupertino"] diff --git a/example/insertFuns.json b/example/insertFuns.json new file mode 100644 index 00000000..024abbab --- /dev/null +++ b/example/insertFuns.json @@ -0,0 +1,85 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 5, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 2000, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "dmeters", + "drop": "yes", + "vgroups": 4, + "duration": "5d", + "keep": "30d", + "pages": 512, + "minrows":5000, + "maxrows":10000, + "stt_trigger":1, + "wal_retention_period": 1, + "wal_retention_size": 10, + "cachemodel": "'both'", + "precision": "ms" + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100000, + "childtable_prefix": "d", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 900000000000000, + "interlace_rows": 1, + "insert_interval": 1000, + "partial_col_num": 2, + "partial_col_from": 3, + "start_timestamp": "now", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"}, + { "type": "INT", "name": "c1", "fun": "count(0,100,1,0) + 100"}, + { "type": "INT", "name": "c2", "fun": "saw(-100,100,20,0) + 1000"}, + { "type": "float", "name": "c3", "fun": "square(0,50,10,3)+20+ 80"}, + { "type": "INT", "name": "c4", "fun": "tri(-20,40,20,5)+50*random(12)+ 10000"} ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/example/insertNow.json b/example/insertNow.json index 8032654d..c839b09a 100644 --- a/example/insertNow.json +++ b/example/insertNow.json @@ -18,17 +18,17 @@ "databases": [ { "dbinfo": { - "name": "smart", + "name": "dmeters", "drop": "yes", "vgroups": 4, - "duration": "10d", - "keep": "100d", + "duration": "5d", + "keep": "30d", "pages": 512, "minrows":5000, "maxrows":10000, "stt_trigger":1, - "wal_retention_period": 10, - "wal_retention_size": 100, + "wal_retention_period": 1, + "wal_retention_size": 10, "cachemodel": "'both'", "precision": "ms" }, @@ -36,7 +36,7 @@ { "name": "meters", "child_table_exists": "no", - "childtable_count": 10000, + "childtable_count": 100000, "childtable_prefix": "d", "auto_create_table": "no", "batch_create_tbl_num": 5, @@ -48,20 +48,20 @@ "interlace_rows": 1, "insert_interval": 1000, "start_timestamp": "now", + "start_fillback_time": "auto", + "timestamp_step": 1000, "sample_format": "csv", "sample_file": "./sample.csv", "use_sample_ts": "no", "tags_file": "", "columns": [ - { - "type": "FLOAT", - "name": "current", - "count": 1, - "max": 12, - "min": 8 - }, - { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, - { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + { "type": "FLOAT", "name": "current", "fun": "3*sin(x)+10*random(2)"}, + { "type": "INT", "name": "voltage", "fun": "40*sin(x)+200*random(10)"}, + { "type": "FLOAT", "name": "phase", "fun": "1*sin(x)+1*random(3)"}, + { "type": "INT", "name": "c1", "fun": "count(0,100,1,0)"}, + { "type": "INT", "name": "c2", "fun": "saw(-100,100,20,0)"}, + { "type": "INT", "name": "c3", "fun": "square(0,60,20,10)"}, + { "type": "INT", "name": "c4", "fun": "tri(-20,100,30,10)"} ], "tags": [ { diff --git a/example/query.json b/example/query.json index ca0a367a..bf74d8da 100644 --- a/example/query.json +++ b/example/query.json @@ -8,7 +8,7 @@ "confirm_parameter_prompt": "no", "continue_if_fail": "yes", "databases": "test", - "query_times": 2, + "query_times": 10, "query_mode": "taosc", "specified_table_query": { "query_interval": 1, diff --git a/example/subscribe.json b/example/subscribe.json index 95e2ac61..aff0ea77 100644 --- a/example/subscribe.json +++ b/example/subscribe.json @@ -1,37 +1,27 @@ { - "filetype": "subscribe", - "cfgdir": "/etc/taos", - "host": "127.0.0.1", - "port": 6030, - "user": "root", - "password": "taosdata", - "databases": "test", - "specified_table_query": { - "concurrent": 1, - "mode": "sync", - "interval": 1000, - "restart": "yes", - "keepProgress": "yes", - "resubAfterConsume": 10, - "sqls": [ - { - "sql": "select current from meters where location = 'beijing';", - "result": "./subscribe_res0.txt" - } - ] - }, - "super_table_query": { - "stblname": "meters", - "threads": 1, - "mode": "sync", - "interval": 1000, - "restart": "yes", - "keepProgress": "yes", - "sqls": [ - { - "sql": "select phase from xxxx where groupid > 3;", - "result": "./subscribe_res1.txt" - } - ] - } + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "result_file": "tmq_res.txt", + "tmq_info": { + "concurrent": 3, + "poll_delay": 100000, + "group.id": "", + "group_mode": "independent", + "create_mode": "parallel", + "client.id": "cliid_0001", + "auto.offset.reset": "earliest", + "enable.manual.commit": "false", + "enable.auto.commit": "false", + "auto.commit.interval.ms": 1000, + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false", + "rows_file": "rowfile", + "topic_list": [ + {"name": "topic1", "sql": "select * from test.meters;"} + ] + } } diff --git a/example/tmq.json b/example/tmq.json index 3573c204..4a7fb423 100644 --- a/example/tmq.json +++ b/example/tmq.json @@ -7,21 +7,21 @@ "password": "taosdata", "result_file": "tmq_res.txt", "tmq_info": { - "concurrent": 99, - "poll_delay": 100000, + "concurrent": 3, + "poll_delay": 10000, "group.id": "", "group_mode": "independent", "create_mode": "parallel", - "client.id": "cliid_0001", + "client.id": "client001", "auto.offset.reset": "earliest", "enable.manual.commit": "false", "enable.auto.commit": "false", "auto.commit.interval.ms": 1000, "experimental.snapshot.enable": "false", "msg.with.table.name": "false", - "rows_file": "", + "rows_file": "rowfile", "topic_list": [ - {"name": "dbtstb_0001", "sql": "select * from dbt.stb;"} + {"name": "topic1", "sql": "select * from test.meters;"} ] } } diff --git a/inc/bench.h b/inc/bench.h index 74f4e226..801dfcfa 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -230,7 +230,7 @@ typedef unsigned __int32 uint32_t; #define BENCH_CHINESE \ "Nchar and binary are basic unicode chinese characters, optional." #define BENCH_NORMAL "Only create normal table without super table, optional." -#define BENCH_RANDOM "Data source is randomly generated, optional." +#define BENCH_RANDOM "Each child table generates different random data, this option need much memory. ( all memory = childs count * prepared_rand)" #define BENCH_AGGR "Query aggregation function after insertion, optional." #define BENCH_YES "Pass confirmation prompt to continue, optional." #define BENCH_RANGE "Range of disordered timestamp, default is 1000." @@ -470,6 +470,7 @@ enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 SUBSCRIBE_TEST, // 2 + CSVFILE_TEST // 3 }; enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT }; @@ -561,8 +562,33 @@ typedef struct SChildField { #define FUNTYPE_NONE 0 #define FUNTYPE_SIN 1 #define FUNTYPE_COS 2 +#define FUNTYPE_COUNT 3 +#define FUNTYPE_SAW 4 +#define FUNTYPE_SQUARE 5 +#define FUNTYPE_TRI 6 -#define FUNTYPE_CNT 2 +#define FUNTYPE_CNT 7 + +#define TAG_BATCH_COUNT 100 + +#define GEN_RANDOM 0 +#define GEN_ORDER 1 + +#define COL_GEN (field->gen == GEN_ORDER ? k : taosRandom()) + +#define tmpInt8(field) tmpInt8Impl(field, 0) +#define tmpUint8(field) tmpUint8Impl(field, 0) +#define tmpInt16(field) tmpInt16Impl(field, 0) +#define tmpUint16(field) tmpUint16Impl(field, 0) + +#define tmpInt32(field) tmpInt32Impl (field,0,0,0) +#define tmpUint32(field) tmpUint32Impl(field,0,0,0) +#define tmpInt64(field) tmpInt64Impl (field,0,0) +#define tmpUint64(field) tmpUint64Impl(field,0,0) +#define tmpFloat(field) tmpFloatImpl (field,0,0,0) +#define tmpDouble(field) tmpDoubleImpl(field,0,0) + +#define COMP_NAME_LEN 32 typedef struct SField { uint8_t type; @@ -578,10 +604,23 @@ typedef struct SField { // fun uint8_t funType; float multiple; - int32_t addend; + float addend; + float base; int32_t random; + int32_t period; + int32_t offset; + int32_t step; + bool sma; + bool fillNull; + uint8_t gen; // see GEN_ define + + // compress + char encode[COMP_NAME_LEN]; + char compress[COMP_NAME_LEN]; + char level[COMP_NAME_LEN]; + } Field; typedef struct STSMA { @@ -608,6 +647,8 @@ typedef struct STSMA { #define SUIT_DATAPOS_MUL_FILE 4 #define SUIT_DATAPOS_MIX 5 +#define VAL_NULL "NULL" + enum CONTINUE_IF_FAIL_MODE { NO_IF_FAILED, // 0 YES_IF_FAILED, // 1 @@ -615,13 +656,17 @@ enum CONTINUE_IF_FAIL_MODE { }; typedef struct SChildTable_S { - char name[TSDB_TABLE_NAME_LEN]; + char* name; bool useOwnSample; char *sampleDataBuf; uint64_t insertRows; BArray *childCols; + int64_t ts; // record child table ts + int32_t pkCur; + int32_t pkCnt; } SChildTable; +#define PRIMARY_KEY "PRIMARY KEY" typedef struct SSuperTable_S { char *stbName; bool random_data_source; // rand_gen or sample @@ -686,10 +731,12 @@ typedef struct SSuperTable_S { uint64_t timestamp_step; uint64_t angle_step; int64_t startTimestamp; + int64_t startFillbackTime; int64_t specifiedColumns; char sampleFile[MAX_FILE_NAME_LEN]; char tagsFile[MAX_FILE_NAME_LEN]; uint32_t partialColNum; + uint32_t partialColFrom; char *partialColNameBuf; BArray *cols; BArray *tags; @@ -701,9 +748,10 @@ typedef struct SSuperTable_S { char *sampleDataBuf; bool useSampleTs; - char *tagDataBuf; bool tcpTransfer; bool non_stop; + bool autoFillback; // "start_fillback_time" item set "auto" + char *calcNow; // need calculate now timestamp expression char *comment; int delay; int file_factor; @@ -713,6 +761,13 @@ typedef struct SSuperTable_S { int ttl; int32_t keep_trying; uint32_t trying_interval; + // primary key + bool primary_key; + int repeat_ts_min; + int repeat_ts_max; + + // execute sqls after create super table + char **sqls; } SSuperTable; typedef struct SDbCfg_S { @@ -736,14 +791,12 @@ typedef struct SSTREAM_S { bool drop; } SSTREAM; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 typedef struct SVGroup_S { int32_t vgId; uint64_t tbCountPerVgId; SChildTable **childTblArray; uint64_t tbOffset; // internal use } SVGroup; -#endif // TD_VER_COMPATIBLE_3_0_0_0 // typedef struct SDataBase_S { char * dbName; @@ -871,7 +924,6 @@ typedef struct SArguments_S { uint32_t binwidth; uint32_t intColumnCount; uint32_t nthreads; - bool nthreads_auto; uint32_t table_threads; uint64_t prepared_rand; uint32_t reqPerReq; @@ -912,6 +964,10 @@ typedef struct SArguments_S { enum CONTINUE_IF_FAIL_MODE continueIfFail; bool mistMode; bool escape_character; + bool pre_load_tb_meta; + char csvPath[MAX_FILE_NAME_LEN]; + + bool bind_vgroup; } SArguments; typedef struct SBenchConn { @@ -944,6 +1000,9 @@ typedef struct SThreadInfo_S { uint64_t totalInsertRows; uint64_t totalQueried; int64_t totalDelay; + int64_t totalDelay1; + int64_t totalDelay2; + int64_t totalDelay3; uint64_t querySeq; TAOS_SUB *tsub; char ** lines; @@ -957,6 +1016,7 @@ typedef struct SThreadInfo_S { char **sml_tags_json_array; char **sml_json_value_array; uint64_t start_time; + uint64_t pos; // point for sampleDataBuff uint64_t max_sql_len; FILE *fp; char filePath[MAX_PATH_LEN]; @@ -976,6 +1036,7 @@ typedef struct SThreadInfo_S { // check sql result char *csql; int32_t clen; // csql current write position + bool stmtBind; } threadInfo; typedef struct SQueryThreadInfo_S { @@ -1095,12 +1156,29 @@ void printVersion(); int32_t benchParseSingleOpt(int32_t key, char* arg); void printErrCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res); -void printWarnCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res); + +int32_t benchGetTotalMemory(int64_t *totalKB); #ifndef LINUX int32_t benchParseArgsNoArgp(int argc, char* argv[]); #endif -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k); +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t* delay3); +// if return true, timestmap must add timestap_step, else timestamp no need changed +bool needChangeTs(SSuperTable * stbInfo, int32_t *pkCur, int32_t *pkCnt); + +// tmp function +bool tmpBool(Field *field); +int8_t tmpInt8Impl(Field *field, int64_t k); +uint8_t tmpUint8Impl(Field *field, int64_t k); +int16_t tmpInt16Impl(Field *field, int64_t k); +uint16_t tmpUint16Impl(Field *field, int64_t k); +int tmpInt32Impl(Field *field, int i, int angle, int32_t k); +uint32_t tmpUint32Impl(Field *field, int i, int angle, int64_t k); +int64_t tmpInt64Impl(Field *field, int32_t angle, int32_t k); +uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k); +float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k); +double tmpDoubleImpl(Field *field, int32_t angle, int32_t k); +int tmpStr(char *tmp, int iface, Field *field, int64_t k); #endif // INC_BENCH_H_ diff --git a/inc/benchCsv.h b/inc/benchCsv.h new file mode 100644 index 00000000..25d0c55e --- /dev/null +++ b/inc/benchCsv.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef INC_BENCHCSV_H_ +#define INC_BENCHCSV_H_ + +#include + +int csvTestProcess(); + +int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir); + +char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k); + +char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k); + +int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); + +void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir); + +int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); +int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); + +#endif // INC_BENCHCSV_H_ diff --git a/inc/benchData.h b/inc/benchData.h index cf42916a..29fed177 100644 --- a/inc/benchData.h +++ b/inc/benchData.h @@ -28,9 +28,10 @@ int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf, int64_t bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag, BArray *childCols); -int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, uint64_t tableSeq); +int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, char* tagData, uint64_t tableSeq); uint32_t bindParamBatch(threadInfo *pThreadInfo, - uint32_t batch, int64_t startTime, SChildTable *childTbl); + uint32_t batch, int64_t startTime, + SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2, int64_t *delay3); int prepareSampleData(SDataBase* database, SSuperTable* stbInfo); void generateSmlJsonTags(tools_cJSON *tagsList, char **sml_tags_json_array, @@ -48,4 +49,9 @@ void generateSmlTaosJsonCols(tools_cJSON *array, uint32_t accumulateRowLen(BArray *fields, int iface); void generateSmlJsonValues( char **sml_tags_json_array, SSuperTable *stbInfo, int tableSeq); + +// generateTag data from random or csv file, cnt is get count for each +bool generateTagData(SSuperTable *stbInfo, char *buf, int64_t cnt, FILE* csv); +// get tag from csv file +FILE* openTagCsv(SSuperTable* stbInfo); #endif // INC_BENCHDATA_H_ diff --git a/inc/benchDataMix.h b/inc/benchDataMix.h index e29afb31..fe352015 100644 --- a/inc/benchDataMix.h +++ b/inc/benchDataMix.h @@ -17,7 +17,7 @@ #define __BENCHDATAMIX_H_ -uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix); +uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal); // data generate by calc ts uint32_t dataGenByCalcTs(Field* fd, char* pstr, uint32_t len, int64_t ts); diff --git a/inc/toolsdef.h b/inc/toolsdef.h index f560dfd0..aaf918b8 100644 --- a/inc/toolsdef.h +++ b/inc/toolsdef.h @@ -234,4 +234,6 @@ void errorPrintReqArg2(char *program, char *wrong_arg); void errorPrintReqArg3(char *program, char *wrong_arg); int setConsoleEcho(bool on); +char *toolsFormatTimestamp(char *buf, int64_t val, int32_t precision); + #endif // __TOOLSDEF_H_ diff --git a/packaging/win/msvcp140d.dll b/packaging/win/msvcp140d.dll new file mode 100644 index 00000000..3b8217f5 Binary files /dev/null and b/packaging/win/msvcp140d.dll differ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb2a20f1..401c69c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -95,16 +95,29 @@ IF(GIT_FOUND) OUTPUT_VARIABLE TAOSDUMP_STATUS ERROR_QUIET ) - EXECUTE_PROCESS( - COMMAND sh "-c" "echo '${TAOSDUMP_FULLTAG}' | awk -F '-' '{print $2}'" - RESULT_VARIABLE RESULT - OUTPUT_VARIABLE TAOSDUMP_TAG - ) - EXECUTE_PROCESS( - COMMAND sh "-c" "echo '${TAOSBENCHMARK_FULLTAG}' | awk -F '-' '{print $2}'" - RESULT_VARIABLE RESULT - OUTPUT_VARIABLE TAOSBENCHMARK_TAG - ) + + # version + IF (DEFINED TD_VER_NUMBER) + # use tdengine version + SET(TAOSBENCHMARK_TAG ${TD_VER_NUMBER}) + SET(TAOSDUMP_TAG ${TD_VER_NUMBER}) + MESSAGE(STATUS "use TD_VER_NUMBER version: " ${TD_VER_NUMBER}) + ELSE () + # use internal version + EXECUTE_PROCESS( + COMMAND sh "-c" "echo '${TAOSDUMP_FULLTAG}' | awk -F '-' '{print $2}'" + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSDUMP_TAG + ) + MESSAGE(STATUS "taosdump use origin version: " ${TAOSDUMP_TAG}) + EXECUTE_PROCESS( + COMMAND sh "-c" "echo '${TAOSBENCHMARK_FULLTAG}' | awk -F '-' '{print $2}'" + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSBENCHMARK_TAG + ) + MESSAGE(STATUS "taosBenchmark use origin version: " ${TAOSBENCHMARK_TAG}) + ENDIF () + EXECUTE_PROCESS( COMMAND sh -c "git --git-dir=${CMAKE_CURRENT_LIST_DIR}/../.git --work-tree=${CMAKE_CURRENT_LIST_DIR}/.. status -z -s ${CMAKE_CURRENT_LIST_DIR}/bench*.c" RESULT_VARIABLE RESULT @@ -114,7 +127,7 @@ IF(GIT_FOUND) IF ("${TAOSDUMP_COMMIT_SHA1}" STREQUAL "") SET(TAOSDUMP_COMMIT_SHA1 "unknown") ELSE () - STRING(SUBSTRING "${TAOSDUMP_COMMIT_SHA1}" 0 7 TAOSDUMP_COMMIT_SHA1) + STRING(SUBSTRING "${TAOSDUMP_COMMIT_SHA1}" 0 40 TAOSDUMP_COMMIT_SHA1) STRING(STRIP "${TAOSDUMP_COMMIT_SHA1}" TAOSDUMP_COMMIT_SHA1) ENDIF () IF ("${TAOSDUMP_TAG}" STREQUAL "") @@ -126,7 +139,7 @@ IF(GIT_FOUND) IF ("${TAOSBENCHMARK_COMMIT_SHA1}" STREQUAL "") SET(TAOSBENCHMARK_COMMIT_SHA1 "unknown") ELSE () - STRING(SUBSTRING "${TAOSBENCHMARK_COMMIT_SHA1}" 0 7 TAOSBENCHMARK_COMMIT_SHA1) + STRING(SUBSTRING "${TAOSBENCHMARK_COMMIT_SHA1}" 0 40 TAOSBENCHMARK_COMMIT_SHA1) STRING(STRIP "${TAOSBENCHMARK_COMMIT_SHA1}" TAOSBENCHMARK_COMMIT_SHA1) ENDIF () IF ("${TAOSBENCHMARK_TAG}" STREQUAL "") @@ -222,9 +235,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEPENDENCIES(taosdump deps-jansson) ADD_DEPENDENCIES(taosdump deps-snappy) IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ELSE () INCLUDE_DIRECTORIES(/usr/local/include) @@ -233,9 +246,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(OS_ID "Darwin") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ENDIF () @@ -426,10 +439,11 @@ ELSE () ADD_DEFINITIONS(-DWINDOWS) SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8") IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ELSE () - ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchDataGeometry.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF () ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c) diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c index 8f1fb171..c6273f39 100644 --- a/src/benchCommandOpt.c +++ b/src/benchCommandOpt.c @@ -28,6 +28,13 @@ extern char g_configDir[MAX_PATH_LEN]; #define TAOSBENCHMARK_STATUS "unknown" #endif +#ifndef TD_PRODUCT_NAME +#define TD_PRODUCT_NAME "TDengine" +#endif + +// libtaos.so +extern char buildinfo[]; + char *g_aggreFuncDemo[] = {"*", "count(*)", "avg(current)", @@ -43,13 +50,15 @@ void printVersion() { char taosBenchmark_ver[] = TAOSBENCHMARK_TAG; char taosBenchmark_commit[] = TAOSBENCHMARK_COMMIT_SHA1; char taosBenchmark_status[] = TAOSBENCHMARK_STATUS; - if (0 == strlen(taosBenchmark_status)) { - printf("version: %s\ngitinfo: %s\n", - taosBenchmark_ver, taosBenchmark_commit); - } else { - printf("version: %s\ngitinfo: %s\nstatus: %s\n", - taosBenchmark_ver, taosBenchmark_commit, taosBenchmark_status); - } + + // version + printf("%s\ntaosBenchmark version: %s\ngit: %s\n", TD_PRODUCT_NAME, taosBenchmark_ver, taosBenchmark_commit); +#ifdef LINUX + printf("build: %s\n ", buildinfo); +#endif + if (strlen(taosBenchmark_status) > 0) { + printf("status: %s\n", taosBenchmark_status); + } } void parseFieldDatatype(char *dataType, BArray *fields, bool isTag) { @@ -237,7 +246,6 @@ void initArgument() { g_arguments->performance_print = 0; g_arguments->output_file = DEFAULT_OUTPUT; g_arguments->nthreads = DEFAULT_NTHREADS; - g_arguments->nthreads_auto = true; g_arguments->table_threads = DEFAULT_NTHREADS; g_arguments->prepared_rand = DEFAULT_PREPARED_RAND; g_arguments->reqPerReq = DEFAULT_REQ_PER_REQ; @@ -304,9 +312,9 @@ void modifyArgument() { } if (superTable->iface == STMT_IFACE) { - if (g_arguments->reqPerReq > INT16_MAX) { - g_arguments->reqPerReq = INT16_MAX; - } + //if (g_arguments->reqPerReq > INT16_MAX) { + // g_arguments->reqPerReq = INT16_MAX; + //} if (g_arguments->prepared_rand > g_arguments->reqPerReq) { g_arguments->prepared_rand = g_arguments->reqPerReq; } @@ -364,11 +372,6 @@ void modifyArgument() { g_arguments->rest_server_ver_major = getServerVersionRest(g_arguments->port); } - - if (g_arguments->demo_mode && TAOSC_IFACE == g_arguments->iface) { - g_arguments->mistMode = true; - g_arguments->prepared_rand = 360; - } } static void *queryStableAggrFunc(void *sarg) { diff --git a/src/benchCsv.c b/src/benchCsv.c new file mode 100644 index 00000000..34e0ba7b --- /dev/null +++ b/src/benchCsv.c @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include +#include + + +// +// main etry +// + +#define SHOW_CNT 100000 + +static void *csvWriteThread(void *param) { + // write thread + for (int i = 0; i < g_arguments->databases->size; i++) { + // database + SDataBase * db = benchArrayGet(g_arguments->databases, i); + for (int j=0; j < db->superTbls->size; j++) { + // stb + SSuperTable* stb = benchArrayGet(db->superTbls, j); + // gen csv + int ret = genWithSTable(db, stb, g_arguments->csvPath); + if(ret != 0) { + errorPrint("failed generate to csv. db=%s stb=%s error code=%d \n", db->dbName, stb->stbName, ret); + return NULL; + } + } + } + return NULL; +} + +int csvTestProcess() { + pthread_t handle; + int ret = pthread_create(&handle, NULL, csvWriteThread, NULL); + if (ret != 0) { + errorPrint("pthread_create failed. error code =%d \n", ret); + return -1; + } + + infoPrint("start output to csv %s ...\n", g_arguments->csvPath); + int64_t start = toolsGetTimestampMs(); + pthread_join(handle, NULL); + int64_t delay = toolsGetTimestampMs() - start; + infoPrint("output to csv %s finished. delay:%"PRId64"s \n", g_arguments->csvPath, delay/1000); + + return 0; +} + +int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { + // filename + int ret = 0; + char outFile[MAX_FILE_NAME_LEN] = {0}; + obtainCsvFile(outFile, db, stb, outDir); + FILE * fs = fopen(outFile, "w"); + if(fs == NULL) { + errorPrint("failed create csv file. file=%s, last errno=%d strerror=%s \n", outFile, errno, strerror(errno)); + return -1; + } + + int rowLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->lenOfCols + stb->tags->size + stb->cols->size; + int bufLen = rowLen * g_arguments->reqPerReq; + char* buf = benchCalloc(1, bufLen, true); + + infoPrint("start write csv file: %s \n", outFile); + + if (stb->interlaceRows > 0) { + // interlace mode + ret = interlaceWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + } else { + // batch mode + ret = batchWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + } + + tmfree(buf); + fclose(fs); + + succPrint("end write csv file: %s \n", outFile); + return ret; +} + + +void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir) { + sprintf(outFile, "%s%s-%s.csv", outDir, db->dbName, stb->stbName); +} + +int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { + size_t size = fwrite(buf, 1, len, f); + if(size != len) { + errorPrint("failed to write csv file. expect write length:%d real write length:%d \n", len, (int32_t)size); + return -1; + } + return 0; +} + +int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { + int ret = 0; + int pos = 0; + int64_t tk = 0; + int64_t show = 0; + + int tagDataLen = stb->lenOfTags + stb->tags->size + 256; + char * tagData = (char *) benchCalloc(1, tagDataLen, true); + int colDataLen = stb->lenOfCols + stb->cols->size + 256; + char * colData = (char *) benchCalloc(1, colDataLen, true); + + // gen child name + for (int64_t i = 0; i < stb->childTblCount; i++) { + int64_t ts = stb->startTimestamp; + int64_t ck = 0; + // tags + genTagData(tagData, stb, i, &tk); + // insert child column data + for(int64_t j = 0; j < stb->insertRows; j++) { + genColumnData(colData, stb, ts, db->precision, &ck); + // combine + pos += sprintf(buf + pos, "%s,%s\n", tagData, colData); + if (bufLen - pos < minRemain) { + // submit + ret = writeCsvFile(fs, buf, pos); + if (ret != 0) { + goto END; + } + + pos = 0; + } + + // ts move next + ts += stb->timestamp_step; + + // check cancel + if(g_arguments->terminate) { + infoPrint("%s", "You are cancel, exiting ...\n"); + ret = -1; + goto END; + } + + // print show + if (++show % SHOW_CNT == 0) { + infoPrint("batch write child table cnt = %"PRId64 " all rows = %" PRId64 "\n", i+1, show); + } + + } + } + + if (pos > 0) { + ret = writeCsvFile(fs, buf, pos); + pos = 0; + } + +END: + // free + tmfree(tagData); + tmfree(colData); + return ret; +} + +int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { + int ret = 0; + int pos = 0; + int64_t n = 0; // already inserted rows for one child table + int64_t tk = 0; + int64_t show = 0; + + char **tagDatas = (char **)benchCalloc(stb->childTblCount, sizeof(char *), true); + int colDataLen = stb->lenOfCols + stb->cols->size + 256; + char * colData = (char *) benchCalloc(1, colDataLen, true); + int64_t last_ts = stb->startTimestamp; + + while (n < stb->insertRows ) { + for (int64_t i = 0; i < stb->childTblCount; i++) { + // start one table + int64_t ts = last_ts; + int64_t ck = 0; + // tags + if (tagDatas[i] == NULL) { + tagDatas[i] = genTagData(NULL, stb, i, &tk); + } + + // calc need insert rows + int64_t needInserts = stb->interlaceRows; + if(needInserts > stb->insertRows - n) { + needInserts = stb->insertRows - n; + } + + for (int64_t j = 0; j < needInserts; j++) { + genColumnData(colData, stb, ts, db->precision, &ck); + // combine tags,cols + pos += sprintf(buf + pos, "%s,%s\n", tagDatas[i], colData); + if (bufLen - pos < minRemain) { + // submit + ret = writeCsvFile(fs, buf, pos); + if (ret != 0) { + goto END; + } + pos = 0; + } + + // ts move next + ts += stb->timestamp_step; + + // check cancel + if(g_arguments->terminate) { + infoPrint("%s", "You are cancel, exiting ... \n"); + ret = -1; + goto END; + } + + // print show + if (++show % SHOW_CNT == 0) { + infoPrint("interlace write child table index = %"PRId64 " all rows = %"PRId64 "\n", i+1, show); + } + } + + // if last child table + if (i + 1 == stb->childTblCount ) { + n += needInserts; + last_ts = ts; + } + } + } + + if (pos > 0) { + ret = writeCsvFile(fs, buf, pos); + pos = 0; + } + +END: + // free + for(int64_t m = 0 ; m < stb->childTblCount; m ++) { + tmfree(tagDatas[m]); + } + tmfree(colData); + return ret; +} + +// gen tag data +char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { + // malloc + char* tagData; + if (buf == NULL) { + int tagDataLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->tags->size + 32; + tagData = benchCalloc(1, tagDataLen, true); + } else { + tagData = buf; + } + + int pos = 0; + // tbname + pos += sprintf(tagData, "\'%s%"PRId64"\'", stb->childTblPrefix, i); + // tags + pos += genRowByField(tagData + pos, stb->tags, stb->tags->size, stb->binaryPrefex, stb->ncharPrefex, k); + + return tagData; +} + +// gen column data +char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k) { + char szTime[128] = {0}; + toolsFormatTimestamp(szTime, ts, precision); + int pos = sprintf(colData, "\'%s\'", szTime); + + // columns + genRowByField(colData + pos, stb->cols, stb->cols->size, stb->binaryPrefex, stb->ncharPrefex, k); + return colData; +} + + +int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k) { + + // other cols data + int32_t pos1 = 0; + for(uint16_t i = 0; i < fieldCnt; i++) { + Field* fd = benchArrayGet(fields, i); + char* prefix = ""; + if(fd->type == TSDB_DATA_TYPE_BINARY || fd->type == TSDB_DATA_TYPE_VARBINARY) { + if(binanryPrefix) { + prefix = binanryPrefix; + } + } else if(fd->type == TSDB_DATA_TYPE_NCHAR) { + if(ncharPrefix) { + prefix = ncharPrefix; + } + } + + pos1 += dataGenByField(fd, buf, pos1, prefix, k, ""); + } + + return pos1; +} diff --git a/src/benchData.c b/src/benchData.c index a01e0cd8..ff522704 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -42,22 +42,106 @@ const char* locations_sml[] = { #include "benchLocations.h" #endif +int32_t funCount(int32_t min, int32_t max, int32_t step, int32_t loop) { + int32_t range = abs(max - min); + int32_t maxCnt = range / step; + int32_t val = min + (loop % maxCnt) * step ; + + return val; +} + +int32_t funSaw(int32_t min, int32_t max, int32_t period, int32_t loop) { + int32_t range = abs(max - min); + int32_t step = range / period; + int32_t val = min + (loop % period) * step ; + return val; +} + +int32_t funSquare(int32_t min, int32_t max, int32_t period, int32_t loop) { + int32_t change = (loop/period) % 2; + if (change) + return min; + else + return max; +} + +int32_t funTriAngle(int32_t min, int32_t max, int32_t period, int32_t loop) { + int32_t range = abs(max - min); + int32_t change = (loop/period) % 2; + int32_t step = range / period; + int32_t cnt = 0; + if(change) + cnt = period - loop % period; + else + cnt = loop % period; + + return min + cnt * step; +} + + // calc expression value like 10*sin(x) + 100 -float calc_expr_value(Field *field, int32_t angle) { +float funValueFloat(Field *field, int32_t angle, int32_t loop) { float radian = ATOR(angle); float funVal = 0; + if (field->funType == FUNTYPE_SIN) funVal = sin(radian); else if (field->funType == FUNTYPE_COS) funVal = cos(radian); + else if (field->funType == FUNTYPE_COUNT) + funVal = (float)funCount(field->min, field->max, field->step, loop); + else if (field->funType == FUNTYPE_SAW) + funVal = (float)funSaw(field->min, field->max, field->period, loop + field->offset ); + else if (field->funType == FUNTYPE_SQUARE) + funVal = (float)funSquare(field->min, field->max, field->period, loop + field->offset); + else if (field->funType == FUNTYPE_TRI) + funVal = (float)funTriAngle(field->min, field->max, field->period, loop + field->offset); + + if(field->multiple != 0) + funVal *= field->multiple; + + if ( field->addend !=0 && field->random > 0 ) { + float rate = taosRandom() % field->random; + funVal += field->addend * (rate/100); + } else if(field->addend !=0 ) { + funVal += field->addend; + } - float val = field->multiple * funVal + field->addend; - if (field->random >0) { + funVal += field->base; + return funVal; +} + +// calc expression value like 10*sin(x) + 100 +int32_t funValueInt32(Field *field, int32_t angle, int32_t loop) { + float radian = ATOR(angle); + int32_t funVal = 0; + + if (field->funType == FUNTYPE_SIN) + funVal = (int32_t)sin(radian); + else if (field->funType == FUNTYPE_COS) + funVal = (int32_t)cos(radian); + else if (field->funType == FUNTYPE_COUNT) + funVal = funCount(field->min, field->max, field->step, loop); + else if (field->funType == FUNTYPE_SAW) + funVal = funSaw(field->min, field->max, field->period, loop + field->offset ); + else if (field->funType == FUNTYPE_SQUARE) + funVal = funSquare(field->min, field->max, field->period, loop + field->offset); + else if (field->funType == FUNTYPE_TRI) + funVal = funTriAngle(field->min, field->max, field->period, loop + field->offset); + + if(field->multiple != 0) + funVal *= field->multiple; + + if ( field->addend !=0 && field->random > 0 ) { float rate = taosRandom() % field->random; - val += field->addend * (rate/100); + funVal += field->addend * (rate/100); + } else if(field->addend !=0 ) { + funVal += field->addend; } - return val; + funVal += field->base; + + return funVal; } @@ -130,7 +214,7 @@ void rand_string(char *str, int size, bool chinese) { } } -int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, uint64_t tableSeq) { +int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, char* tagData, uint64_t tableSeq) { int len = 0; char *prepare = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); int n; @@ -143,7 +227,7 @@ int prepareStmt(SSuperTable *stbInfo, TAOS_STMT *stmt, uint64_t tableSeq) { TSDB_MAX_ALLOWED_SQL_LEN - len, "INSERT INTO ? USING `%s` TAGS (%s) %s VALUES(?", stbInfo->stbName, - stbInfo->tagDataBuf + stbInfo->lenOfTags * tableSeq, + tagData + stbInfo->lenOfTags * tableSeq, ttl); } else { n = snprintf(prepare + len, TSDB_MAX_ALLOWED_SQL_LEN - len, @@ -205,19 +289,21 @@ static bool getSampleFileNameByPattern(char *filePath, return true; } -static int generateSampleFromCsv(char *buffer, - char *file, int32_t length, - int64_t size) { +static int generateSampleFromCsv(char *buffer, char* file, FILE* fp, int32_t length, int64_t size) { size_t n = 0; char * line = NULL; int getRows = 0; + bool needClose = false; - FILE *fp = fopen(file, "r"); - if (fp == NULL) { - errorPrint("Failed to open sample file: %s, reason:%s\n", file, - strerror(errno)); - return -1; + if (file != NULL && fp == NULL) { + fp = fopen(file, "r"); + if (fp == NULL) { + errorPrint("open csv file failed. file=%s\n", file); + return -1; + } + needClose = true; } + while (1) { ssize_t readLen = 0; #if defined(WIN32) || defined(WIN64) @@ -229,9 +315,9 @@ static int generateSampleFromCsv(char *buffer, if (-1 == readLen) { #endif if (0 != fseek(fp, 0, SEEK_SET)) { - errorPrint("Failed to fseek file: %s, reason:%s\n", - file, strerror(errno)); - fclose(fp); + errorPrint("Failed to fseek , reason:%s\n", strerror(errno)); + if(needClose) + fclose(fp); return -1; } continue; @@ -261,7 +347,10 @@ static int generateSampleFromCsv(char *buffer, } } - fclose(fp); + if(needClose) { + fclose(fp); + } + tmfree(line); return 0; } @@ -299,6 +388,8 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { Field *field = benchArrayGet(fields, i); switch (field->type) { case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: len += field->length + 3; @@ -354,20 +445,9 @@ uint32_t accumulateRowLen(BArray *fields, int iface) { return len; } -static int tmpStr(char *tmp, int iface, Field *field, int i) { - if (g_arguments->demo_mode) { - unsigned int tmpRand = taosRandom(); - if (g_arguments->chinese) { - snprintf(tmp, field->length, "%s", - locations_chinese[tmpRand % 10]); - } else if (SML_IFACE == iface) { - snprintf(tmp, field->length, "%s", - locations_sml[tmpRand % 10]); - } else { - snprintf(tmp, field->length, "%s", - locations[tmpRand % 10]); - } - } else if (field->values) { + +int tmpStr(char *tmp, int iface, Field *field, int64_t k) { + if (field->values) { int arraySize = tools_cJSON_GetArraySize(field->values); if (arraySize) { tools_cJSON *buf = tools_cJSON_GetArrayItem( @@ -381,70 +461,29 @@ static int tmpStr(char *tmp, int iface, Field *field, int i) { __func__, arraySize); return -1; } - } else { - rand_string(tmp, field->length, - g_arguments->chinese); - } - return 0; -} - -static int tmpGeometry(char *tmp, int iface, Field *field, int i) { - if (g_arguments->demo_mode) { - // TODO - } else if (field->values) { - int arraySize = tools_cJSON_GetArraySize(field->values); - if (arraySize) { - tools_cJSON *buf = tools_cJSON_GetArrayItem(field->values, taosRandom() % arraySize); - snprintf(tmp, field->length, "%s", buf->valuestring); + } else if (g_arguments->demo_mode) { + unsigned int tmpRand = taosRandom(); + if (g_arguments->chinese) { + snprintf(tmp, field->length, "%s", + locations_chinese[tmpRand % 10]); + } else if (SML_IFACE == iface) { + snprintf(tmp, field->length, "%s", + locations_sml[tmpRand % 10]); } else { - errorPrint( - "%s() cannot read correct value " - "from json file. array size: %d\n", - __func__, arraySize); - return -1; + snprintf(tmp, field->length, "%s", + locations[tmpRand % 10]); } } else { - int maxType = getGeoMaxType(field->length); - rand_geometry(tmp, field->length, maxType); + if(field->gen == GEN_ORDER) { + snprintf(tmp, field->length, "%"PRId64, k); + } else { + rand_string(tmp, taosRandom() % field->length, g_arguments->chinese); + } } return 0; } -FORCE_INLINE double tmpDoubleImpl(Field *field, int32_t angle) { - double doubleTmp = (double)(field->min); - - if(field->funType != FUNTYPE_NONE) { - doubleTmp = calc_expr_value(field, angle); - } else if (field->max != field->min) { - doubleTmp += ((taosRandom() % - (field->max - field->min)) + - taosRandom() % 1000000 / 1000000.0); - } - return doubleTmp; -} - -FORCE_INLINE double tmpDouble(Field *field) { - return tmpDoubleImpl(field, 0); -} - - -FORCE_INLINE uint64_t tmpUint64(Field *field) { - uint64_t ubigintTmp = field->min; - if (field->max != field->min) { - ubigintTmp += (taosRandom() % (field->max - field->min)); - } - return ubigintTmp; -} - -FORCE_INLINE uint32_t tmpUint(Field *field) { - uint32_t uintTmp = field->min; - if (field->max != field->min) { - uintTmp += (taosRandom() % (field->max - field->min)); - } - return uintTmp; -} - -FORCE_INLINE bool tmpBool(Field *field) { +bool tmpBool(Field *field) { bool boolTmp; if (field->min == field->max) { boolTmp = (field->min)?1:0; @@ -454,70 +493,115 @@ FORCE_INLINE bool tmpBool(Field *field) { return boolTmp; } -FORCE_INLINE int8_t tmpInt8(Field *field) { +int8_t tmpInt8Impl(Field *field, int64_t k) { int8_t tinyint = field->min; if (field->min != field->max) { - tinyint += (taosRandom() % (field->max - field->min)); + tinyint += COL_GEN % (field->max - field->min); } return tinyint; } -FORCE_INLINE uint8_t tmpUint8(Field *field) { +uint8_t tmpUint8Impl(Field *field, int64_t k) { uint8_t utinyint = field->min; if (field->min != field->max) { - utinyint += (taosRandom() % (field->max - field->min)); + utinyint += (COL_GEN % (field->max - field->min)); } return utinyint; } -FORCE_INLINE int16_t tmpInt16(Field *field) { +int16_t tmpInt16Impl(Field *field, int64_t k) { int16_t smallint = field->min; if (field->min != field->max) { - smallint += (taosRandom() % (field->max - field->min)); + smallint += (COL_GEN % (field->max - field->min)); } return smallint; } -FORCE_INLINE uint16_t tmpUint16(Field *field) { +uint16_t tmpUint16Impl(Field *field, int64_t k) { uint16_t usmallintTmp = field->min; if (field->max != field->min) { - usmallintTmp += (taosRandom() % (field->max - field->min)); + usmallintTmp += (COL_GEN % (field->max - field->min)); } return usmallintTmp; } -FORCE_INLINE int64_t tmpInt64Impl(Field *field, int32_t angle) { +int tmpInt32Impl(Field *field, int i, int angle, int32_t k) { + int intTmp; + if (field->funType != FUNTYPE_NONE) { + // calc from function + intTmp = funValueInt32(field, angle, k); + } else if ((g_arguments->demo_mode) && (i == 0)) { + unsigned int tmpRand = taosRandom(); + intTmp = tmpRand % 10 + 1; + } else if ((g_arguments->demo_mode) && (i == 1)) { + intTmp = 105 + taosRandom() % 10; + } else { + if (field->min < (-1 * (RAND_MAX >> 1))) { + field->min = -1 * (RAND_MAX >> 1); + } + if (field->max > (RAND_MAX >> 1)) { + field->max = RAND_MAX >> 1; + } + intTmp = field->min; + if (field->max != field->min) { + intTmp += (COL_GEN % (field->max - field->min)); + } + } + return intTmp; +} + +uint32_t tmpUint32Impl(Field *field, int i, int angle, int64_t k) { + uint32_t intTmp; + if (field->funType != FUNTYPE_NONE) { + // calc from function + intTmp = funValueInt32(field, angle, k); + } else if ((g_arguments->demo_mode) && (i == 0)) { + unsigned int tmpRand = taosRandom(); + intTmp = tmpRand % 10 + 1; + } else if ((g_arguments->demo_mode) && (i == 1)) { + intTmp = 105 + taosRandom() % 10; + } else { + intTmp = field->min; + if (field->max != field->min) { + intTmp += (COL_GEN % (field->max - field->min)); + } + } + return intTmp; +} + +int64_t tmpInt64Impl(Field *field, int32_t angle, int32_t k) { int64_t bigintTmp = field->min; if(field->funType != FUNTYPE_NONE) { - bigintTmp = calc_expr_value(field, angle); + bigintTmp = funValueInt32(field, angle, k); } else if (field->min != field->max) { - bigintTmp += (taosRandom() % (field->max - field->min)); + bigintTmp += (COL_GEN % (field->max - field->min)); } return bigintTmp; } -FORCE_INLINE int64_t tmpInt64(Field *field) { - return tmpInt64Impl(field, 0); -} - -FORCE_INLINE float tmpFloat(Field *field) { - float floatTmp = field->min; - if (field->max != field->min) { - floatTmp += ((taosRandom() % (field->max - field->min)) - + taosRandom() % 1000 / 1000.0); +uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k) { + uint64_t bigintTmp = field->min; + if(field->funType != FUNTYPE_NONE) { + bigintTmp = funValueInt32(field, angle, k); + } else if (field->min != field->max) { + bigintTmp += (COL_GEN % (field->max - field->min)); } - return floatTmp; + return bigintTmp; } -static float tmpFloatImpl(Field *field, int i, int32_t angle) { +float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k) { float floatTmp = (float)field->min; if(field->funType != FUNTYPE_NONE) { - floatTmp = calc_expr_value(field, angle); + floatTmp = funValueFloat(field, angle, k); } else { if (field->max != field->min) { - floatTmp += ((taosRandom() % - (field->max - field->min)) - + (taosRandom() % 1000) / 1000.0); + if (field->gen == GEN_ORDER) { + floatTmp += (k % (field->max - field->min)); + } else { + floatTmp += ((taosRandom() % + (field->max - field->min)) + + (taosRandom() % 1000) / 1000.0); + } } if (g_arguments->demo_mode && i == 0) { floatTmp = (float)(9.8 + 0.04 * (taosRandom() % 10) @@ -530,37 +614,20 @@ static float tmpFloatImpl(Field *field, int i, int32_t angle) { return floatTmp; } -static float tmpFloatI(Field *field, int i) { - return tmpFloatImpl(field, i, 0); -} - -static int tmpInt32Impl(Field *field, int i, int angle) { - int intTmp; - if (field->funType != FUNTYPE_NONE) { - // calc from function - intTmp = calc_expr_value(field, angle); - } else if ((g_arguments->demo_mode) && (i == 0)) { - unsigned int tmpRand = taosRandom(); - intTmp = tmpRand % 10 + 1; - } else if ((g_arguments->demo_mode) && (i == 1)) { - intTmp = 105 + taosRandom() % 10; - } else { - if (field->min < (-1 * (RAND_MAX >> 1))) { - field->min = -1 * (RAND_MAX >> 1); - } - if (field->max > (RAND_MAX >> 1)) { - field->max = RAND_MAX >> 1; - } - intTmp = field->min; - if (field->max != field->min) { - intTmp += (taosRandom() % (field->max - field->min)); +double tmpDoubleImpl(Field *field, int32_t angle, int32_t k) { + double doubleTmp = (double)(field->min); + if(field->funType != FUNTYPE_NONE) { + doubleTmp = funValueFloat(field, angle, k); + } else if (field->max != field->min) { + if(field->gen == GEN_ORDER) { + doubleTmp += k % (field->max - field->min); + } else { + doubleTmp += ((taosRandom() % + (field->max - field->min)) + + taosRandom() % 1000000 / 1000000.0); } } - return intTmp; -} - -static int tmpInt32(Field *field, int i) { - return tmpInt32Impl(field, i, 0); + return doubleTmp; } static int tmpJson(char *sampleDataBuf, @@ -659,70 +726,71 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_TINYINT: { - int8_t tinyint = tmpInt8(field); + int8_t tinyint = tmpInt8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", tinyint); break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t utinyint = tmpUint8(field); + uint8_t utinyint = tmpUint8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", utinyint); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t smallint = tmpInt16(field); + int16_t smallint = tmpInt16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", smallint); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t usmallint = tmpUint16(field); + uint16_t usmallint = tmpUint16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", usmallint); break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32Impl(field, i, angle); + int32_t intTmp = tmpInt32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", intTmp); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64Impl(field, angle); + int64_t bigintTmp = tmpInt64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRId64",", bigintTmp); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t uintTmp = tmpUint(field); + uint32_t uintTmp = tmpUint32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", uintTmp); break; } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - uint64_t ubigintTmp = tmpUint64(field); + uint64_t ubigintTmp = tmpUint64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRIu64",", ubigintTmp); break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatImpl(field, i, angle); + float floatTmp = tmpFloatImpl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", floatTmp); break; } case TSDB_DATA_TYPE_DOUBLE: { - double double_ = tmpDoubleImpl(field, angle); + double double_ = tmpDoubleImpl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", double_); break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); - if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { + if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } @@ -772,10 +840,10 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, static int fillStmt( SSuperTable *stbInfo, char *sampleDataBuf, - int bufLen, + int64_t bufLen, int lenOfOneRow, BArray *fields, int64_t loop, bool tag, BArray *childCols) { - // fillStmt() + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -785,7 +853,7 @@ static int fillStmt( if (childCols) { childCol = benchArrayGet(childCols, i); } - int n = 0; + int64_t n = 0; switch (field->type) { case TSDB_DATA_TYPE_BOOL: { bool boolTmp = tmpBool(field); @@ -799,7 +867,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_TINYINT: { - int8_t tinyintTmp = tmpInt8(field); + int8_t tinyintTmp = tmpInt8Impl(field, k); if (childCol) { ((int8_t *)childCol->stmtData.data)[k] = tinyintTmp; } else { @@ -810,7 +878,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t utinyintTmp = tmpUint8(field); + uint8_t utinyintTmp = tmpUint8Impl(field, k); if (childCol) { ((uint8_t *)childCol->stmtData.data)[k] = utinyintTmp; } else { @@ -821,7 +889,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t smallintTmp = tmpInt16(field); + int16_t smallintTmp = tmpInt16Impl(field, k); if (childCol) { ((int16_t *)childCol->stmtData.data)[k] = smallintTmp; } else { @@ -832,7 +900,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t usmallintTmp = tmpUint16(field); + uint16_t usmallintTmp = tmpUint16Impl(field, k); if (childCol) { ((uint16_t *)childCol->stmtData.data)[k] = usmallintTmp; } else { @@ -843,7 +911,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle, k); if (childCol) { ((int32_t *)childCol->stmtData.data)[k] = intTmp; } else { @@ -854,7 +922,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle, k); if (childCol) { ((int64_t *)childCol->stmtData.data)[k] = bigintTmp; } else { @@ -865,7 +933,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_UINT: { - uint32_t uintTmp = tmpUint(field); + uint32_t uintTmp = tmpUint32Impl(field, i, angle, k); if (childCol) { ((uint32_t *)childCol->stmtData.data)[k] = uintTmp; } else { @@ -877,7 +945,7 @@ static int fillStmt( } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - uint64_t ubigintTmp = tmpUint64(field); + uint64_t ubigintTmp = tmpUint64Impl(field, angle, k); if (childCol) { ((uint64_t *)childCol->stmtData.data)[k] = ubigintTmp; } else { @@ -888,7 +956,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle, k); if (childCol) { ((float *)childCol->stmtData.data)[k] = floatTmp; } else { @@ -899,7 +967,7 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_DOUBLE: { - double doubleTmp = tmpDouble(field); + double doubleTmp = tmpDoubleImpl(field, angle, k); if (childCol) { ((double *)childCol->stmtData.data)[k] = doubleTmp; } else { @@ -910,9 +978,10 @@ static int fillStmt( break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); - if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { + if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } @@ -949,7 +1018,13 @@ static int fillStmt( } } skip_stmt: - *(sampleDataBuf + pos - 1) = 0; + if (pos > 0) + *(sampleDataBuf + pos - 1) = 0; + angle += stbInfo->timestamp_step/stbInfo->angle_step; + if (angle > 360) { + angle -= 360; + } + } return 0; } @@ -1010,6 +1085,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, int bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -1033,7 +1109,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_TINYINT: { - int8_t tinyint = tmpInt8(field); + int8_t tinyint = tmpInt8Impl(field, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di8 ", field->name, tinyint); @@ -1044,7 +1120,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t utinyint = tmpUint8(field); + uint8_t utinyint = tmpUint8Impl(field, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%uu8 ", field->name, utinyint); @@ -1055,7 +1131,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t smallint = tmpInt16(field); + int16_t smallint = tmpInt16Impl(field, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di16 ", field->name, smallint); @@ -1066,7 +1142,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t usmallint = tmpUint16(field); + uint16_t usmallint = tmpUint16Impl(field, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%uu16 ", field->name, usmallint); @@ -1077,7 +1153,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di32 ", @@ -1090,7 +1166,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%"PRId64"i64 ", @@ -1102,10 +1178,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_UINT: { - uint32_t uintTmp = field->min; - if (field->max != field->min) { - uintTmp += (taosRandom() % (field->max - field->min)); - } + uint32_t uintTmp = tmpUint32Impl(field, i, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, @@ -1120,7 +1193,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - uint64_t ubigintTmp = tmpUint64(field); + uint64_t ubigintTmp = tmpUint64Impl(field, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%"PRIu64"u64 ", @@ -1132,7 +1205,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%ff32 ", field->name, floatTmp); @@ -1143,7 +1216,7 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_DOUBLE: { - double double_ = tmpDouble(field); + double double_ = tmpDoubleImpl(field, angle, k); if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%ff64 ", field->name, double_); @@ -1154,13 +1227,14 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); - if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { + if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } - if (field->type == TSDB_DATA_TYPE_BINARY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) { if (tag) { n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=L\"%s\" ", @@ -1216,6 +1290,11 @@ static int generateRandDataSmlTelnet(SSuperTable *stbInfo, char *sampleDataBuf, } skip_telnet: *(sampleDataBuf + pos - 1) = 0; + angle += stbInfo->timestamp_step/stbInfo->angle_step; + if (angle > 360) { + angle -= 360; + } + } return 0; @@ -1225,6 +1304,7 @@ static int generateRandDataSmlJson(SSuperTable *stbInfo, char *sampleDataBuf, int bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int fieldsSize = fields->size; @@ -1239,70 +1319,71 @@ static int generateRandDataSmlJson(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_TINYINT: { - int8_t tinyint = tmpInt8(field); + int8_t tinyint = tmpInt8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", tinyint); break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t utinyint = tmpUint8(field); + uint8_t utinyint = tmpUint8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", utinyint); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t smallint = tmpInt16(field); + int16_t smallint = tmpInt16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", smallint); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t usmallint = tmpUint16(field); + uint16_t usmallint = tmpUint16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", usmallint); break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%d,", intTmp); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRId64",", bigintTmp); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t uintTmp = tmpUint(field); + uint32_t uintTmp = tmpUint32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%u,", uintTmp); break; } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - uint64_t ubigintTmp = tmpUint64(field); + uint64_t ubigintTmp = tmpUint64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%"PRIu64",", ubigintTmp); break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", floatTmp); break; } case TSDB_DATA_TYPE_DOUBLE: { - double double_ = tmpDouble(field); + double double_ = tmpDoubleImpl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%f,", double_); break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); - if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { + if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } @@ -1329,6 +1410,10 @@ static int generateRandDataSmlJson(SSuperTable *stbInfo, char *sampleDataBuf, } skip_json: *(sampleDataBuf + pos - 1) = 0; + angle += stbInfo->timestamp_step/stbInfo->angle_step; + if (angle > 360) { + angle -= 360; + } } return 0; @@ -1338,6 +1423,7 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, int bufLen, int lenOfOneRow, BArray * fields, int64_t loop, bool tag) { + int angle = stbInfo->startTimestamp % 360; // 0 ~ 360 for (int64_t k = 0; k < loop; ++k) { int64_t pos = k * lenOfOneRow; int n = 0; @@ -1365,73 +1451,74 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, break; } case TSDB_DATA_TYPE_TINYINT: { - int8_t tinyint = tmpInt8(field); + int8_t tinyint = tmpInt8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di8,", field->name, tinyint); break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t utinyint = tmpUint8(field); + uint8_t utinyint = tmpUint8Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%uu8,", field->name, utinyint); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t smallint = tmpInt16(field); + int16_t smallint = tmpInt16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di16,", field->name, smallint); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t usmallint = tmpUint16(field); + uint16_t usmallint = tmpUint16Impl(field, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%uu16,", field->name, usmallint); break; } case TSDB_DATA_TYPE_INT: { - int32_t intTmp = tmpInt32(field, i); + int32_t intTmp = tmpInt32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%di32,", field->name, intTmp); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t bigintTmp = tmpInt64(field); + int64_t bigintTmp = tmpInt64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%"PRId64"i64,", field->name, bigintTmp); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t uintTmp = tmpUint(field); + uint32_t uintTmp = tmpUint32Impl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%uu32,", field->name, uintTmp); break; } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - uint64_t ubigintTmp = tmpUint64(field); + uint64_t ubigintTmp = tmpUint64Impl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%"PRIu64"u64,", field->name, ubigintTmp); break; } case TSDB_DATA_TYPE_FLOAT: { - float floatTmp = tmpFloatI(field, i); + float floatTmp = tmpFloatImpl(field, i, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%ff32,", field->name, floatTmp); break; } case TSDB_DATA_TYPE_DOUBLE: { - double doubleTmp = tmpDouble(field); + double doubleTmp = tmpDoubleImpl(field, angle, k); n = snprintf(sampleDataBuf + pos, bufLen - pos, "%s=%ff64,", field->name, doubleTmp); break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *tmp = benchCalloc(1, field->length + 1, false); - if (0 != tmpStr(tmp, stbInfo->iface, field, i)) { + if (0 != tmpStr(tmp, stbInfo->iface, field, k)) { free(tmp); return -1; } @@ -1472,6 +1559,11 @@ static int generateRandDataSmlLine(SSuperTable *stbInfo, char *sampleDataBuf, } skip_line: *(sampleDataBuf + pos - 1) = 0; + angle += stbInfo->timestamp_step/stbInfo->angle_step; + if (angle > 360) { + angle -= 360; + } + } return 0; @@ -1548,9 +1640,17 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { if (stbInfo->partialColNum != 0 && ((stbInfo->iface == TAOSC_IFACE || stbInfo->iface == REST_IFACE))) { - if (stbInfo->partialColNum > stbInfo->cols->size) { - stbInfo->partialColNum = stbInfo->cols->size; - } else { + // check valid + if(stbInfo->partialColFrom >= stbInfo->cols->size) { + stbInfo->partialColFrom = 0; + infoPrint("stbInfo->partialColFrom(%d) is large than stbInfo->cols->size(%"PRIu64") \n ",stbInfo->partialColFrom,stbInfo->cols->size); + } + + if (stbInfo->partialColFrom + stbInfo->partialColNum > stbInfo->cols->size) { + stbInfo->partialColNum = stbInfo->cols->size - stbInfo->partialColFrom ; + } + + if(stbInfo->partialColNum < stbInfo->cols->size) { stbInfo->partialColNameBuf = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); int pos = 0; @@ -1564,7 +1664,7 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { } else { pos += n; } - for (int i = 0; i < stbInfo->partialColNum; ++i) { + for (int i = stbInfo->partialColFrom; i < stbInfo->partialColFrom + stbInfo->partialColNum; ++i) { Field * col = benchArrayGet(stbInfo->cols, i); n = snprintf(stbInfo->partialColNameBuf+pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, @@ -1576,9 +1676,14 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { pos += n; } } - for (int i = 0; i < stbInfo->partialColNum; ++i) { + + // first part set noen + for (uint32_t i = 0; i < stbInfo->partialColFrom; ++i) { + Field * col = benchArrayGet(stbInfo->cols, i); + col->none = true; } - for (int i = stbInfo->partialColNum; i < stbInfo->cols->size; ++i) { + // last part set none + for (uint32_t i = stbInfo->partialColFrom + stbInfo->partialColNum; i < stbInfo->cols->size; ++i) { Field * col = benchArrayGet(stbInfo->cols, i); col->none = true; } @@ -1597,6 +1702,10 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { stbInfo->stbName, stbInfo->lenOfCols, g_arguments->prepared_rand); if (stbInfo->random_data_source) { if (g_arguments->mistMode) { + infoPrint("Each child table using different random prepare data pattern. need " + "all memory(%d M) = childs(%"PRId64") * prepared_rand(%"PRId64") * lenOfCols(%d) \n", + (int32_t)(stbInfo->childTblCount*g_arguments->prepared_rand*stbInfo->lenOfCols/1024/1024), + stbInfo->childTblCount, g_arguments->prepared_rand, stbInfo->lenOfCols); for (int64_t child = 0; child < stbInfo->childTblCount; child++) { SChildTable *childTbl = stbInfo->childTblArray[child]; if (STMT_IFACE == stbInfo->iface) { @@ -1636,7 +1745,7 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { } } if (generateSampleFromCsv(stbInfo->sampleDataBuf, - stbInfo->sampleFile, stbInfo->lenOfCols, + stbInfo->sampleFile, NULL, stbInfo->lenOfCols, g_arguments->prepared_rand)) { errorPrint("Failed to generate sample from csv file %s\n", stbInfo->sampleFile); @@ -1672,6 +1781,7 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { if (generateSampleFromCsv( childTbl->sampleDataBuf, sampleFilePath, + NULL, stbInfo->lenOfCols, g_arguments->prepared_rand)) { errorPrint("Failed to generate sample from file " @@ -1688,35 +1798,6 @@ int prepareSampleData(SDataBase* database, SSuperTable* stbInfo) { } } - if (stbInfo->tags->size != 0) { - stbInfo->tagDataBuf = - benchCalloc( - 1, stbInfo->childTblCount*stbInfo->lenOfTags, true); - infoPrint( - "generate stable<%s> tags data with lenOfTags<%u> * " - "childTblCount<%" PRIu64 ">\n", - stbInfo->stbName, stbInfo->lenOfTags, - stbInfo->childTblCount); - if (stbInfo->tagsFile[0] != 0) { - if (generateSampleFromCsv( - stbInfo->tagDataBuf, stbInfo->tagsFile, - stbInfo->lenOfTags, - stbInfo->childTblCount)) { - return -1; - } - } else { - if (generateRandData(stbInfo, - stbInfo->tagDataBuf, - stbInfo->childTblCount*stbInfo->lenOfTags, - stbInfo->lenOfTags, - stbInfo->tags, - stbInfo->childTblCount, true, NULL)) { - return -1; - } - } - debugPrint("tagDataBuf: %s\n", stbInfo->tagDataBuf); - } - if (0 != convertServAddr( stbInfo->iface, stbInfo->tcpTransfer, @@ -1740,80 +1821,89 @@ int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, uint32_t bindParamBatch(threadInfo *pThreadInfo, uint32_t batch, int64_t startTime, - SChildTable *childTbl) { + SChildTable *childTbl, int32_t *pkCur, int32_t *pkCnt, int32_t *n, int64_t *delay2, int64_t *delay3) { TAOS_STMT *stmt = pThreadInfo->conn->stmt; SSuperTable *stbInfo = pThreadInfo->stbInfo; uint32_t columnCount = stbInfo->cols->size; - memset(pThreadInfo->bindParams, 0, - (sizeof(TAOS_MULTI_BIND) * (columnCount + 1))); - memset(pThreadInfo->is_null, 0, batch); - - for (int c = 0; c < columnCount + 1; c++) { - TAOS_MULTI_BIND *param = - (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + - sizeof(TAOS_MULTI_BIND) * c); - char data_type; - if (c == 0) { - data_type = TSDB_DATA_TYPE_TIMESTAMP; - param->buffer_length = sizeof(int64_t); - param->buffer = pThreadInfo->bind_ts_array; - } else { - Field *col = benchArrayGet(stbInfo->cols, c - 1); - data_type = col->type; - if (childTbl->useOwnSample) { - ChildField *childCol = benchArrayGet(childTbl->childCols, c-1); - param->buffer = childCol->stmtData.data; - param->is_null = childCol->stmtData.is_null; + + if (!pThreadInfo->stmtBind) { + pThreadInfo->stmtBind = true; + memset(pThreadInfo->bindParams, 0, + (sizeof(TAOS_MULTI_BIND) * (columnCount + 1))); + memset(pThreadInfo->is_null, 0, batch); + + for (int c = 0; c < columnCount + 1; c++) { + TAOS_MULTI_BIND *param = + (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + + sizeof(TAOS_MULTI_BIND) * c); + char data_type; + if (c == 0) { + data_type = TSDB_DATA_TYPE_TIMESTAMP; + param->buffer_length = sizeof(int64_t); + param->buffer = pThreadInfo->bind_ts_array; } else { - param->buffer = col->stmtData.data; - param->is_null = col->stmtData.is_null; + Field *col = benchArrayGet(stbInfo->cols, c - 1); + data_type = col->type; + if (childTbl->useOwnSample) { + ChildField *childCol = benchArrayGet(childTbl->childCols, c-1); + param->buffer = childCol->stmtData.data; + param->is_null = childCol->stmtData.is_null; + } else { + param->buffer = col->stmtData.data; + param->is_null = col->stmtData.is_null; + } + param->buffer_length = col->length; + debugPrint("col[%d]: type: %s, len: %d\n", c, + convertDatatypeToString(data_type), + col->length); } - param->buffer_length = col->length; - debugPrint("col[%d]: type: %s, len: %d\n", c, - convertDatatypeToString(data_type), - col->length); - } - param->buffer_type = data_type; - param->length = benchCalloc(batch, sizeof(int32_t), true); + param->buffer_type = data_type; + param->length = benchCalloc(batch, sizeof(int32_t), true); - for (int b = 0; b < batch; b++) { - param->length[b] = (int32_t)param->buffer_length; + for (int b = 0; b < batch; b++) { + param->length[b] = (int32_t)param->buffer_length; + } + param->num = batch; } - param->num = batch; } + // set ts array values for (uint32_t k = 0; k < batch; k++) { /* columnCount + 1 (ts) */ if (stbInfo->disorderRatio) { *(pThreadInfo->bind_ts_array + k) = - startTime + getTSRandTail(stbInfo->timestamp_step, k, + startTime + getTSRandTail(stbInfo->timestamp_step, *n, stbInfo->disorderRatio, stbInfo->disorderRange); } else { - *(pThreadInfo->bind_ts_array + k) = - startTime + stbInfo->timestamp_step * k; + *(pThreadInfo->bind_ts_array + k) = startTime + stbInfo->timestamp_step * (*n); + } + + // check n need add + if (!stbInfo->primary_key || needChangeTs(stbInfo, pkCur, pkCnt)) { + *n = *n + 1; } } + int64_t start = toolsGetTimestampUs(); if (taos_stmt_bind_param_batch( stmt, (TAOS_MULTI_BIND *)pThreadInfo->bindParams)) { errorPrint("taos_stmt_bind_param_batch() failed! reason: %s\n", taos_stmt_errstr(stmt)); return 0; } - - for (int c = 0; c < stbInfo->cols->size + 1; c++) { - TAOS_MULTI_BIND *param = - (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + - sizeof(TAOS_MULTI_BIND) * c); - tmfree(param->length); - } - - // if msg > 3MB, break - if (taos_stmt_add_batch(stmt)) { - errorPrint("taos_stmt_add_batch() failed! reason: %s\n", - taos_stmt_errstr(stmt)); - return 0; + *delay2 += toolsGetTimestampUs() - start; + + if(stbInfo->autoTblCreating) { + start = toolsGetTimestampUs(); + if (taos_stmt_add_batch(pThreadInfo->conn->stmt) != 0) { + errorPrint("taos_stmt_add_batch() failed! reason: %s\n", + taos_stmt_errstr(pThreadInfo->conn->stmt)); + return 0; + } + if(delay3) { + *delay3 += toolsGetTimestampUs() - start; + } } return batch; } @@ -1848,10 +1938,11 @@ void generateSmlJsonTags(tools_cJSON *tagsList, } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(tag->length + 1, 1, false); rand_string(buf, tag->length, g_arguments->chinese); - if (tag->type == TSDB_DATA_TYPE_BINARY) { + if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(tags, tagName, buf); } else { tools_cJSON_AddStringToObject(tags, tagName, buf); @@ -1914,10 +2005,11 @@ void generateSmlTaosJsonTags(tools_cJSON *tagsList, SSuperTable *stbInfo, } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(tag->length + 1, 1, false); rand_string(buf, tag->length, g_arguments->chinese); - if (tag->type == TSDB_DATA_TYPE_BINARY) { + if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(tagObj, "value", buf); tools_cJSON_AddStringToObject(tagObj, "type", "binary"); } else { @@ -1973,6 +2065,7 @@ void generateSmlJsonValues( break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); @@ -2015,6 +2108,7 @@ void generateSmlJsonCols(tools_cJSON *array, tools_cJSON *tag, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); @@ -2072,10 +2166,11 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: { char *buf = (char *)benchCalloc(col->length + 1, 1, false); rand_string(buf, col->length, g_arguments->chinese); - if (col->type == TSDB_DATA_TYPE_BINARY) { + if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_VARBINARY) { tools_cJSON_AddStringToObject(value, "value", buf); tools_cJSON_AddStringToObject(value, "type", "binary"); } else { @@ -2102,3 +2197,40 @@ void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, tools_cJSON_AddStringToObject(record, "metric", stbInfo->stbName); tools_cJSON_AddItemToArray(array, record); } + +// generateTag data from random or csv file +bool generateTagData(SSuperTable *stbInfo, char *buf, int64_t cnt, FILE* csv) { + if(csv) { + if (generateSampleFromCsv( + buf, NULL, csv, + stbInfo->lenOfTags, + cnt)) { + return false; + } + } else { + if (generateRandData(stbInfo, + buf, + cnt * stbInfo->lenOfTags, + stbInfo->lenOfTags, + stbInfo->tags, + cnt, true, NULL)) { + errorPrint("Generate Tag Rand Data Failed. stb=%s\n", stbInfo->stbName); + return false; + } + } + + return true; +} + +// open tag from csv file +FILE* openTagCsv(SSuperTable* stbInfo) { + FILE* csvFile = NULL; + if (stbInfo->tagsFile[0] != 0) { + csvFile = fopen(stbInfo->tagsFile, "r"); + if (csvFile == NULL) { + errorPrint("Failed to open sample file: %s, reason:%s\n", stbInfo->tagsFile, strerror(errno)); + return NULL; + } + } + return csvFile; +} diff --git a/src/benchDataMix.c b/src/benchDataMix.c index 1169667e..117b2c48 100644 --- a/src/benchDataMix.c +++ b/src/benchDataMix.c @@ -14,8 +14,6 @@ #include "benchDataMix.h" #include -#define VAL_NULL "NULL" - #define VBOOL_CNT 3 int32_t inul = 20; // interval null count @@ -95,68 +93,75 @@ uint32_t genRadomString(char* val, uint32_t len, char* prefix) { return size; } + // data row generate by randowm -uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix) { +uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal) { uint32_t size = 0; - char val[512] = VAL_NULL; - if( RD(inul) == 0 ) { - size = sprintf(pstr + len, ",%s", VAL_NULL); + int64_t nowts= 0; + char val[512] = {0}; + if( fd->fillNull && RD(inul) == 0 ) { + size = sprintf(pstr + len, ",%s", nullVal); return size; } + const char * format = ",%s"; + switch (fd->type) { case TSDB_DATA_TYPE_BOOL: - strcpy(val, RD(2) ? "true" : "false"); + sprintf(val, "%d", tmpBool(fd)); break; // timestamp case TSDB_DATA_TYPE_TIMESTAMP: - strcpy(val, "now"); + nowts = toolsGetTimestampMs(); + strcpy(val, "\'"); + toolsFormatTimestamp(val, nowts, TSDB_TIME_PRECISION_MILLI); + strcat(val, "\'"); break; // signed case TSDB_DATA_TYPE_TINYINT: - SIGNED_RANDOM(int8_t, 0xFF, "%d") + sprintf(val, "%d", tmpInt8Impl(fd, *k)); break; case TSDB_DATA_TYPE_SMALLINT: - SIGNED_RANDOM(int16_t, 0xFFFF, "%d") + sprintf(val, "%d", tmpInt16Impl(fd, *k)); break; case TSDB_DATA_TYPE_INT: - SIGNED_RANDOM(int32_t, 0xFFFFFFFF, "%d") + sprintf(val, "%d", tmpInt32Impl(fd, 0, 0, *k)); break; case TSDB_DATA_TYPE_BIGINT: - SIGNED_RANDOM(int64_t, 0xFFFFFFFFFFFFFFFF, "%"PRId64) + sprintf(val, "%"PRId64, tmpInt64Impl(fd, 0, *k)); break; // unsigned case TSDB_DATA_TYPE_UTINYINT: - UNSIGNED_RANDOM(uint8_t, 0xFF,"%u") + sprintf(val, "%u", tmpUint8Impl(fd, *k)); break; case TSDB_DATA_TYPE_USMALLINT: - UNSIGNED_RANDOM(uint16_t, 0xFFFF, "%u") + sprintf(val, "%u", tmpUint16Impl(fd, *k)); break; case TSDB_DATA_TYPE_UINT: - UNSIGNED_RANDOM(uint32_t, 0xFFFFFFFF, "%u") + sprintf(val, "%u", tmpUint32Impl(fd, 0, 0, *k)); break; case TSDB_DATA_TYPE_UBIGINT: - UNSIGNED_RANDOM(uint64_t, 0xFFFFFFFFFFFFFFFF, "%"PRIu64) + sprintf(val, "%"PRIu64, tmpUint64Impl(fd, 0, *k)); break; // float double case TSDB_DATA_TYPE_FLOAT: - FLOAT_RANDOM(float, FLT_MIN, FLT_MAX) + sprintf(val, "%f", tmpFloatImpl(fd, 0, 0, *k)); break; case TSDB_DATA_TYPE_DOUBLE: - FLOAT_RANDOM(double, DBL_MIN, DBL_MAX) + sprintf(val, "%f", tmpDoubleImpl(fd, 0, *k)); break; // binary nchar - case TSDB_DATA_TYPE_BINARY: - genRadomString(val, fd->length > sizeof(val) ? sizeof(val) : fd->length, prefix); - break; case TSDB_DATA_TYPE_NCHAR: - genRadomString(val, fd->length > sizeof(val) ? sizeof(val) : fd->length, prefix); + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: + format = ",\'%s\'"; + tmpStr(val, 0, fd, *k); break; default: break; } - size += sprintf(pstr + len, ",%s", val); + size += sprintf(pstr + len, format, val); return size; } @@ -199,6 +204,7 @@ uint32_t dataGenByCalcTs(Field* fd, char* pstr, uint32_t len, int64_t ts) { break; // binary nchar case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: sprintf(val, "%" PRId64, ts); break; diff --git a/src/benchInsert.c b/src/benchInsert.c index f5d871fd..e7246dcb 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -11,6 +11,7 @@ */ #include +#include "wrapDb.h" #include #include @@ -72,7 +73,6 @@ static int getSuperTableFromServerTaosc( res = taos_query(conn->taos, command); int32_t code = taos_errno(res); if (code != 0) { - printWarnCmdCodeStr(command, code, res); infoPrint("stable %s does not exist, will create one\n", stbInfo->stbName); closeBenchConn(conn); @@ -217,12 +217,27 @@ static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) { } #endif // WEBSOCKET +int getCompressStr(Field* col, char* buf) { + int pos = 0; + if(strlen(col->encode) > 0) { + pos +=sprintf(buf + pos, "encode \'%s\' ", col->encode); + } + if(strlen(col->compress) > 0) { + pos +=sprintf(buf + pos, "compress \'%s\' ", col->compress); + } + if(strlen(col->level) > 0) { + pos +=sprintf(buf + pos, "level \'%s\' ", col->level); + } + + return pos; +} + static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { if (g_arguments->supplementInsert) { return 0; } - uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size; + uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15 + COMP_NAME_LEN*3) * stbInfo->cols->size; char *colsBuf = benchCalloc(1, col_buffer_len, false); char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); int len = 0; @@ -246,6 +261,20 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { ",%s %s", col->name, convertDatatypeToString(col->type)); } + + // primary key + if(stbInfo->primary_key && colIndex == 0) { + len += n; + n = snprintf(colsBuf + len, col_buffer_len - len, " %s", PRIMARY_KEY); + } + + // compress key + char keys[COMP_NAME_LEN*3] = ""; + if (getCompressStr(col, keys) > 0) { + len += n; + n = snprintf(colsBuf + len, col_buffer_len - len, " %s", keys); + } + if (n < 0 || n >= col_buffer_len - len) { errorPrint("%s() LN%d, snprintf overflow on %d\n", __func__, __LINE__, colIndex); @@ -330,8 +359,8 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { int length = snprintf( command, TSDB_MAX_ALLOWED_SQL_LEN, g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" - : "CREATE TABLE %s.%s (ts TIMESTAMP%s) TAGS %s", + ? "CREATE TABLE IF NOT EXISTS `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" + : "CREATE TABLE IF NOT EXISTS %s.%s (ts TIMESTAMP%s) TAGS %s", database->dbName, stbInfo->stbName, colsBuf, tagsBuf); tmfree(colsBuf); tmfree(tagsBuf); @@ -411,7 +440,7 @@ static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { return ret; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 + int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { int vgroups = 0; char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; @@ -474,13 +503,36 @@ int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { return vgroups; } -#endif // TD_VER_COMPATIBLE_3_0_0_0 + +int32_t toolsGetDefaultVGroups() { + int32_t cores = toolsGetNumberOfCores(); + if (cores < 3 ) { + return 1; + } + + int64_t MemKB = 0; + benchGetTotalMemory(&MemKB); + + infoPrint("check local machine CPU: %d Memory:%d MB \n", cores, (int32_t)(MemKB/1024)); + if (MemKB <= 2*1024*1024) { // 2G + return 1; + } else if (MemKB <= 4*1024*1024) { // 4G + return 2; + } else if (MemKB <= 8*1024*1024) { // 8G + return 3; + } else if (MemKB <= 16*1024*1024) { // 16G + return 4; + } else if (MemKB <= 32*1024*1024) { // 32G + return 5; + } else { + return cores / 2; + } +} int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { int dataLen = 0; int n; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { + if (-1 != g_arguments->inputted_vgroups) { n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, g_arguments->escape_character ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" @@ -496,12 +548,7 @@ int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); } -#else - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); -#endif // TD_VER_COMPATIBLE_3_0_0_0 + if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { errorPrint("%s() LN%d snprintf overflow\n", __func__, __LINE__); @@ -634,27 +681,29 @@ int32_t getRemainVnodes(SBenchConn *conn) { int createDatabaseTaosc(SDataBase* database) { char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + // conn SBenchConn* conn = initBenchConn(); if (NULL == conn) { return -1; } - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->streams->size; i++) { - SSTREAM* stream = benchArrayGet(g_arguments->streams, i); - if (stream->drop) { - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DROP STREAM IF EXISTS %s;", - stream->stream_name); - if (queryDbExecCall(conn, command)) { - closeBenchConn(conn); - return -1; - } - infoPrint("%s\n", command); - memset(command, 0, SHORT_1K_SQL_BUFF_LEN); + + // drop stream in old database + for (int i = 0; i < g_arguments->streams->size; i++) { + SSTREAM* stream = benchArrayGet(g_arguments->streams, i); + if (stream->drop) { + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DROP STREAM IF EXISTS %s;", + stream->stream_name); + if (queryDbExecCall(conn, command)) { + closeBenchConn(conn); + return -1; } + infoPrint("%s\n", command); + memset(command, 0, SHORT_1K_SQL_BUFF_LEN); } } + // drop old database snprintf(command, SHORT_1K_SQL_BUFF_LEN, g_arguments->escape_character ? "DROP DATABASE IF EXISTS `%s`;": @@ -674,9 +723,9 @@ int createDatabaseTaosc(SDataBase* database) { #endif } + // get remain vgroups int remainVnodes = INT_MAX; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { + if (g_arguments->bind_vgroup) { remainVnodes = getRemainVnodes(conn); if (0 >= remainVnodes) { errorPrint("Remain vnodes %d, failed to create database\n", @@ -684,9 +733,9 @@ int createDatabaseTaosc(SDataBase* database) { return -1; } } -#endif - geneDbCreateCmd(database, command, remainVnodes); + // generate and execute create database sql + geneDbCreateCmd(database, command, remainVnodes); int32_t code = queryDbExecCall(conn, command); int32_t trying = g_arguments->keep_trying; while (code && trying) { @@ -719,19 +768,17 @@ int createDatabaseTaosc(SDataBase* database) { } infoPrint("command to create database: <%s>\n", command); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->superTbls) { - if (g_arguments->nthreads_auto) { - int32_t vgroups = getVgroupsOfDb(conn, database); - if (vgroups <=0) { - closeBenchConn(conn); - errorPrint("Database %s's vgroups is %d\n", - database->dbName, vgroups); - return -1; - } + + // malloc and get vgroup + if (g_arguments->bind_vgroup) { + int32_t vgroups = getVgroupsOfDb(conn, database); + if (vgroups <= 0) { + closeBenchConn(conn); + errorPrint("Database %s's vgroups is %d\n", + database->dbName, vgroups); + return -1; } } -#endif // TD_VER_COMPATIBLE_3_0_0_0 closeBenchConn(conn); return 0; @@ -760,12 +807,12 @@ int createDatabase(SDataBase* database) { } static int generateChildTblName(int len, char *buffer, SDataBase *database, - SSuperTable *stbInfo, uint64_t i, + SSuperTable *stbInfo, uint64_t tableSeq, char* tagData, int i, char *ttl) { if (0 == len) { memset(buffer, 0, TSDB_MAX_ALLOWED_SQL_LEN); len += snprintf(buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE "); + TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE IF NOT EXISTS "); } len += snprintf( @@ -773,9 +820,9 @@ static int generateChildTblName(int len, char *buffer, SDataBase *database, g_arguments->escape_character ? "`%s`.`%s%" PRIu64 "` USING `%s`.`%s` TAGS (%s) %s " : "%s.%s%" PRIu64 " USING %s.%s TAGS (%s) %s ", - database->dbName, stbInfo->childTblPrefix, i, database->dbName, + database->dbName, stbInfo->childTblPrefix, tableSeq, database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + tagData + i * stbInfo->lenOfTags, ttl); return len; } @@ -810,36 +857,44 @@ static int getIntervalOfTblCreating(threadInfo *pThreadInfo, return 0; } +// table create thread static void *createTable(void *sarg) { if (g_arguments->supplementInsert) { return NULL; } - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SDataBase *database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + uint64_t lastTotalCreate = 0; + uint64_t lastPrintTime = toolsGetTimestampMs(); + int32_t len = 0; + int32_t batchNum = 0; + char ttl[SMALL_BUFF_LEN] = ""; + #ifdef LINUX prctl(PR_SET_NAME, "createTable"); #endif - uint64_t lastPrintTime = toolsGetTimestampMs(); pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - int batchNum = 0; infoPrint( "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 "\n", pThreadInfo->threadID, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - - char ttl[SMALL_BUFF_LEN] = ""; if (stbInfo->ttl != 0) { snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); } + // tag read from csv + FILE *csvFile = openTagCsv(stbInfo); + // malloc + char* tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); + int w = 0; // record tagData + int smallBatchCount = 0; - for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; - (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) - && !g_arguments->terminate); i++) { + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to && !g_arguments->terminate; + i++) { if (g_arguments->terminate) { goto create_table_end; } @@ -847,15 +902,15 @@ static void *createTable(void *sarg) { if (stbInfo->childTblCount == 1) { snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", + ? "CREATE TABLE IF NOT EXISTS `%s`.`%s` %s;" + : "CREATE TABLE IF NOT EXISTS %s.%s %s;", database->dbName, stbInfo->stbName, stbInfo->colsOfCreateChildTable); } else { snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", + ? "CREATE TABLE IF NOT EXISTS `%s`.`%s` %s;" + : "CREATE TABLE IF NOT EXISTS %s.%s %s;", database->dbName, stbInfo->childTblArray[i]->name, stbInfo->colsOfCreateChildTable); @@ -865,8 +920,20 @@ static void *createTable(void *sarg) { if (0 == len) { batchNum = 0; } + // generator + if (w == 0) { + if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { + goto create_table_end; + } + } + len = generateChildTblName(len, pThreadInfo->buffer, - database, stbInfo, i, ttl); + database, stbInfo, i, tagData, w, ttl); + // move next + if (++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } batchNum++; smallBatchCount++; @@ -926,10 +993,11 @@ static void *createTable(void *sarg) { batchNum = 0; uint64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { - infoPrint( - "thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - lastPrintTime = currentPrintTime; + float speed = (pThreadInfo->tables_created - lastTotalCreate) * 1000 / (currentPrintTime - lastPrintTime); + infoPrint("thread[%d] already created %" PRId64 " tables, peroid speed: %.0f tables/s\n", + pThreadInfo->threadID, pThreadInfo->tables_created, speed); + lastPrintTime = currentPrintTime; + lastTotalCreate = pThreadInfo->tables_created; } } @@ -957,40 +1025,44 @@ static void *createTable(void *sarg) { pThreadInfo->threadID, pThreadInfo->tables_created); } create_table_end: + // free + tmfree(tagData); tmfree(pThreadInfo->buffer); pThreadInfo->buffer = NULL; + if(csvFile) { + fclose(csvFile); + } return NULL; } -static int startMultiThreadCreateChildTable( - SDataBase* database, SSuperTable* stbInfo) { - int code = -1; - int threads = g_arguments->table_threads; - int64_t ntables; +static int startMultiThreadCreateChildTable(SDataBase* database, SSuperTable* stbInfo) { + int32_t code = -1; + int32_t threads = g_arguments->table_threads; + int64_t ntables; if (stbInfo->childTblTo > 0) { ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + } else if(stbInfo->childTblFrom > 0) { + ntables = stbInfo->childTblCount - stbInfo->childTblFrom; } else { ntables = stbInfo->childTblCount; } pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); - uint64_t tableFrom = 0; + uint64_t tableFrom = stbInfo->childTblFrom; if (threads < 1) { threads = 1; } - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - if (ntables == 0) { - errorPrint("failed to create child table, childTblCount: %"PRId64"\n", - ntables); + errorPrint("failed to create child table, childTblCount: %"PRId64"\n", ntables); goto over; } - int64_t b = ntables % threads; + + int64_t div = ntables / threads; + if (div < 1) { + threads = (int)ntables; + div = 1; + } + int64_t mod = ntables % threads; int threadCnt = 0; for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { @@ -1011,10 +1083,12 @@ static int startMultiThreadCreateChildTable( } } pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + pThreadInfo->ntables = i < mod ? div + 1 : div; + pThreadInfo->end_table_to = i < mod ? tableFrom + div : tableFrom + div - 1; tableFrom = pThreadInfo->end_table_to + 1; pThreadInfo->tables_created = 0; + debugPrint("div table by thread. i=%d from=%"PRId64" to=%"PRId64" ntable=%"PRId64"\n", i, pThreadInfo->start_table_from, + pThreadInfo->end_table_to, pThreadInfo->ntables); pthread_create(pids + i, NULL, createTable, pThreadInfo); threadCnt ++; } @@ -1053,7 +1127,7 @@ static int createChildTables() { "start creating %" PRId64 " table(s) with %d thread(s)\n", g_arguments->totalChildTables, g_arguments->table_threads); } - double start = (double)toolsGetTimestampMs(); + int64_t start = (double)toolsGetTimestampMs(); for (int i = 0; (i < g_arguments->databases->size && !g_arguments->terminate); i++) { @@ -1084,14 +1158,20 @@ static int createChildTables() { } } - double end = (double)toolsGetTimestampMs(); + int64_t end = toolsGetTimestampMs(); + if(end == start) { + end += 1; + } succPrint( "Spent %.4f seconds to create %" PRId64 - " table(s) with %d thread(s), already exist %" PRId64 + " table(s) with %d thread(s) speed: %.0f tables/s, already exist %" PRId64 " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 " table(s) will be auto created\n", - (end - start) / 1000.0, g_arguments->totalChildTables, - g_arguments->table_threads, g_arguments->existedChildTables, + (float)(end - start) / 1000.0, + g_arguments->totalChildTables, + g_arguments->table_threads, + g_arguments->actualChildTables * 1000 / (float)(end - start), + g_arguments->existedChildTables, g_arguments->actualChildTables, g_arguments->autoCreatedChildTables); return 0; @@ -1116,6 +1196,7 @@ static void freeChildTable(SChildTable *childTbl, int colsSize) { } void postFreeResource() { + infoPrint("%s\n", "free resource and exit ..."); if (!g_arguments->terminate) { tmfclose(g_arguments->fpOfInsertResult); } @@ -1139,8 +1220,6 @@ void postFreeResource() { stbInfo->colsOfCreateChildTable = NULL; tmfree(stbInfo->sampleDataBuf); stbInfo->sampleDataBuf = NULL; - tmfree(stbInfo->tagDataBuf); - stbInfo->tagDataBuf = NULL; tmfree(stbInfo->partialColNameBuf); stbInfo->partialColNameBuf = NULL; benchArrayDestroy(stbInfo->batchTblCreatingNumbersArray); @@ -1165,6 +1244,7 @@ void postFreeResource() { child++) { SChildTable *childTbl = stbInfo->childTblArray[child]; + tmfree(childTbl->name); if (childTbl) { freeChildTable(childTbl, stbInfo->cols->size); } @@ -1175,21 +1255,28 @@ void postFreeResource() { tmfree(stbInfo->childTblArray); stbInfo->childTblArray = NULL; benchArrayDestroy(stbInfo->tsmas); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { + + // free sqls + if(stbInfo->sqls) { + char **sqls = stbInfo->sqls; + while (*sqls) { + free(*sqls); + sqls++; + } + tmfree(stbInfo->sqls); + } + + // thread_bind + if (database->vgArray) { for (int32_t v = 0; v < database->vgroups; v++) { SVGroup *vg = benchArrayGet(database->vgArray, v); tmfree(vg->childTblArray); vg->childTblArray = NULL; } + benchArrayDestroy(database->vgArray); + database->vgArray = NULL; } -#endif // TD_VER_COMPATIBLE_3_0_0_0 } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->vgArray) - benchArrayDestroy(database->vgArray); -#endif // TD_VER_COMPATIBLE_3_0_0_0 benchArrayDestroy(database->superTbls); } } @@ -1198,12 +1285,13 @@ void postFreeResource() { tools_cJSON_Delete(root); } -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t *delay3) { SDataBase * database = pThreadInfo->dbInfo; SSuperTable *stbInfo = pThreadInfo->stbInfo; TAOS_RES * res = NULL; int32_t code = 0; uint16_t iface = stbInfo->iface; + int64_t start = 0; int32_t trying = (stbInfo->keep_trying)? stbInfo->keep_trying:g_arguments->keep_trying; @@ -1257,6 +1345,20 @@ int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { break; case STMT_IFACE: + // add batch + if(!stbInfo->autoTblCreating) { + start = toolsGetTimestampUs(); + if (taos_stmt_add_batch(pThreadInfo->conn->stmt) != 0) { + errorPrint("taos_stmt_add_batch() failed! reason: %s\n", + taos_stmt_errstr(pThreadInfo->conn->stmt)); + return -1; + } + if(delay3) { + *delay3 += toolsGetTimestampUs() - start; + } + } + + // execute code = taos_stmt_execute(pThreadInfo->conn->stmt); if (code) { errorPrint( @@ -1366,6 +1468,7 @@ int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { static int smartContinueIfFail(threadInfo *pThreadInfo, SChildTable *childTbl, + char *tagData, int64_t i, char *ttl) { SDataBase * database = pThreadInfo->dbInfo; @@ -1375,11 +1478,11 @@ static int smartContinueIfFail(threadInfo *pThreadInfo, snprintf( buffer, TSDB_MAX_ALLOWED_SQL_LEN, g_arguments->escape_character ? - "CREATE TABLE `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " - : "CREATE TABLE %s.%s USING %s.%s TAGS (%s) %s ", + "CREATE TABLE IF NOT EXISTS `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " + : "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (%s) %s ", database->dbName, childTbl->name, database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + tagData + i * stbInfo->lenOfTags, ttl); debugPrint("creating table: %s\n", buffer); int ret; if (REST_IFACE == stbInfo->iface) { @@ -1449,6 +1552,40 @@ static int64_t getDisorderTs(SSuperTable *stbInfo, int *disorderRange) { return disorderTs; } +void loadChildTableInfo(threadInfo* pThreadInfo) { + SSuperTable *stbInfo = pThreadInfo->stbInfo; + if(!g_arguments->pre_load_tb_meta) { + return ; + } + if(pThreadInfo->conn == NULL) { + return ; + } + + char *db = pThreadInfo->dbInfo->dbName; + int64_t cnt = pThreadInfo->end_table_to - pThreadInfo->start_table_from; + + // 100k + int bufLen = 100 * 1024; + char *buf = benchCalloc(1, bufLen, false); + int pos = 0; + infoPrint("start load child tables(%"PRId64") info...\n", cnt); + int64_t start = toolsGetTimestampUs(); + for(int64_t i = pThreadInfo->start_table_from; i < pThreadInfo->end_table_to; i++) { + SChildTable *childTbl = stbInfo->childTblArray[i]; + pos += sprintf(buf + pos, ",%s.%s", db, childTbl->name); + + if(pos >= bufLen - 256 || i + 1 == pThreadInfo->end_table_to) { + taos_load_table_info(pThreadInfo->conn, buf); + pos = 0; + } + } + int64_t delay = toolsGetTimestampUs() - start; + infoPrint("end load child tables info. delay=%.2fs\n", delay/1E6); + pThreadInfo->totalDelay += delay; + + tmfree(buf); +} + static void *syncWriteInterlace(void *sarg) { threadInfo * pThreadInfo = (threadInfo *)sarg; SDataBase * database = pThreadInfo->dbInfo; @@ -1461,7 +1598,6 @@ static void *syncWriteInterlace(void *sarg) { int64_t insertRows = stbInfo->insertRows; int32_t interlaceRows = stbInfo->interlaceRows; - int64_t pos = 0; uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; uint64_t lastPrintTime = toolsGetTimestampMs(); uint64_t lastTotalInsertRows = 0; @@ -1470,6 +1606,26 @@ static void *syncWriteInterlace(void *sarg) { uint64_t tableSeq = pThreadInfo->start_table_from; int disorderRange = stbInfo->disorderRange; + loadChildTableInfo(pThreadInfo); + // check if filling back mode + bool fillBack = false; + if(stbInfo->useNow && stbInfo->startFillbackTime) { + fillBack = true; + pThreadInfo->start_time = stbInfo->startFillbackTime; + infoPrint("start time change to startFillbackTime = %"PRId64" \n", pThreadInfo->start_time); + } + + FILE* csvFile = NULL; + char* tagData = NULL; + int w = 0; + if (stbInfo->autoTblCreating) { + csvFile = openTagCsv(stbInfo); + tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); + } + int64_t delay1 = 0; + int64_t delay2 = 0; + int64_t delay3 = 0; + while (insertRows > 0) { int64_t tmp_total_insert_rows = 0; uint32_t generated = 0; @@ -1480,13 +1636,24 @@ static void *syncWriteInterlace(void *sarg) { if (g_arguments->terminate) { goto free_of_interlace; } - int64_t timestamp = pThreadInfo->start_time; - SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; - char * tableName = - stbInfo->childTblArray[tableSeq]->name; + int64_t pos = pThreadInfo->pos; + + // get childTable + SChildTable *childTbl; + if (g_arguments->bind_vgroup) { + childTbl = pThreadInfo->vg->childTblArray[tableSeq]; + } else { + childTbl = stbInfo->childTblArray[tableSeq]; + } + + char * tableName = childTbl->name; char *sampleDataBuf = childTbl->useOwnSample? childTbl->sampleDataBuf: stbInfo->sampleDataBuf; + // init ts + if(childTbl->ts == 0) { + childTbl->ts = pThreadInfo->start_time; + } char ttl[SMALL_BUFF_LEN] = ""; if (stbInfo->ttl != 0) { snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); @@ -1505,6 +1672,15 @@ static void *syncWriteInterlace(void *sarg) { if (i == 0) { ds_add_str(&pThreadInfo->buffer, STR_INSERT_INTO); } + + // generator + if (stbInfo->autoTblCreating && w == 0) { + if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { + goto free_of_interlace; + } + } + + // create child table if (stbInfo->partialColNum == stbInfo->cols->size) { if (stbInfo->autoTblCreating) { ds_add_strs(&pThreadInfo->buffer, 8, @@ -1512,8 +1688,7 @@ static void *syncWriteInterlace(void *sarg) { " USING `", stbInfo->stbName, "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, + tagData + stbInfo->lenOfTags * w, ") ", ttl, " VALUES "); } else { ds_add_strs(&pThreadInfo->buffer, 2, @@ -1528,8 +1703,7 @@ static void *syncWriteInterlace(void *sarg) { ") USING `", stbInfo->stbName, "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, + tagData + stbInfo->lenOfTags * w, ") ", ttl, " VALUES "); } else { ds_add_strs(&pThreadInfo->buffer, 4, @@ -1540,22 +1714,43 @@ static void *syncWriteInterlace(void *sarg) { } } + // move next + if (stbInfo->autoTblCreating && ++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } + + // write child data with interlaceRows for (int64_t j = 0; j < interlaceRows; j++) { int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); + + // change fillBack mode with condition + if(fillBack) { + int64_t tsnow = toolsGetTimestamp(database->precision); + if(childTbl->ts >= tsnow){ + fillBack = false; + infoPrint("fillBack mode set end. because timestamp(%"PRId64") >= now(%"PRId64")\n", childTbl->ts, tsnow); + } + } + + // timestamp char time_string[BIGINT_BUFF_LEN]; - if(stbInfo->useNow && stbInfo->interlaceRows == 1) { + if(stbInfo->useNow && stbInfo->interlaceRows == 1 && !fillBack) { snprintf(time_string, BIGINT_BUFF_LEN, "now"); } else { snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", - disorderTs?disorderTs:timestamp); + disorderTs?disorderTs:childTbl->ts); } + + // combine rows timestamp | other cols = sampleDataBuf[pos] ds_add_strs(&pThreadInfo->buffer, 5, "(", time_string, ",", sampleDataBuf + pos * stbInfo->lenOfCols, ") "); + // check buffer enough if (ds_len(pThreadInfo->buffer) > stbInfo->max_sql_len) { errorPrint("sql buffer length (%"PRIu64") " @@ -1565,12 +1760,23 @@ static void *syncWriteInterlace(void *sarg) { stbInfo->max_sql_len); goto free_of_interlace; } + + // move next generated++; pos++; if (pos >= g_arguments->prepared_rand) { pos = 0; } - timestamp += stbInfo->timestamp_step; + if(stbInfo->primary_key) + debugPrint("add child=%s %"PRId64" pk cur=%d cnt=%d \n", childTbl->name, childTbl->ts, childTbl->pkCur, childTbl->pkCnt); + + // primary key + if (!stbInfo->primary_key || needChangeTs(stbInfo, &childTbl->pkCur, &childTbl->pkCnt)) { + childTbl->ts += stbInfo->timestamp_step; + if(stbInfo->primary_key) + debugPrint("changedTs child=%s %"PRId64" pk cur=%d cnt=%d \n", childTbl->name, childTbl->ts, childTbl->pkCur, childTbl->pkCnt); + } + } break; } @@ -1583,6 +1789,7 @@ static void *syncWriteInterlace(void *sarg) { snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", tableName); } + int64_t start = toolsGetTimestampUs(); if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, escapedTbName)) { errorPrint( @@ -1592,9 +1799,13 @@ static void *syncWriteInterlace(void *sarg) { g_fail = true; goto free_of_interlace; } - generated = - bindParamBatch(pThreadInfo, interlaceRows, - timestamp, childTbl); + delay1 += toolsGetTimestampUs() - start; + + int32_t n = 0; + generated = bindParamBatch(pThreadInfo, interlaceRows, + childTbl->ts, childTbl, &childTbl->pkCur, &childTbl->pkCnt, &n, &delay2, &delay3); + + childTbl->ts += stbInfo->timestamp_step * n; break; } case SML_REST_IFACE: @@ -1613,7 +1824,7 @@ static void *syncWriteInterlace(void *sarg) { generateSmlJsonCols( pThreadInfo->json_array, tag, stbInfo, database->sml_precision, - disorderTs?disorderTs:timestamp); + disorderTs?disorderTs:childTbl->ts); } else if (SML_JSON_TAOS_FORMAT == protocol) { tools_cJSON *tag = tools_cJSON_Duplicate( tools_cJSON_GetArrayItem( @@ -1624,7 +1835,7 @@ static void *syncWriteInterlace(void *sarg) { generateSmlTaosJsonCols( pThreadInfo->json_array, tag, stbInfo, database->sml_precision, - disorderTs?disorderTs:timestamp); + disorderTs?disorderTs:childTbl->ts); } else if (TSDB_SML_LINE_PROTOCOL == protocol) { snprintf( pThreadInfo->lines[generated], @@ -1634,20 +1845,23 @@ static void *syncWriteInterlace(void *sarg) { ->sml_tags[(int)tableSeq - pThreadInfo->start_table_from], sampleDataBuf + pos * stbInfo->lenOfCols, - disorderTs?disorderTs:timestamp); + disorderTs?disorderTs:childTbl->ts); } else { snprintf( pThreadInfo->lines[generated], stbInfo->lenOfCols + stbInfo->lenOfTags, "%s %" PRId64 " %s %s", stbInfo->stbName, - disorderTs?disorderTs:timestamp, + disorderTs?disorderTs:childTbl->ts, sampleDataBuf + pos * stbInfo->lenOfCols, pThreadInfo ->sml_tags[(int)tableSeq - pThreadInfo->start_table_from]); } generated++; - timestamp += stbInfo->timestamp_step; + // primary key + if (!stbInfo->primary_key || needChangeTs(stbInfo, &childTbl->pkCur, &childTbl->pkCnt)) { + childTbl->ts += stbInfo->timestamp_step; + } } if (TSDB_SML_JSON_PROTOCOL == protocol || SML_JSON_TAOS_FORMAT == protocol) { @@ -1658,16 +1872,21 @@ static void *syncWriteInterlace(void *sarg) { break; } } + + // move to next table in one batch tableSeq++; tmp_total_insert_rows += interlaceRows; if (tableSeq > pThreadInfo->end_table_to) { + // one tables loop timestamp and pos add tableSeq = pThreadInfo->start_table_from; - pThreadInfo->start_time += - interlaceRows * stbInfo->timestamp_step; + // save + pThreadInfo->pos = pos; if (!stbInfo->non_stop) { insertRows -= interlaceRows; } - if (stbInfo->insert_interval > 0) { + + // if fillBack mode , can't sleep + if (stbInfo->insert_interval > 0 && !fillBack) { debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", __func__, __LINE__, stbInfo->insert_interval); perfPrint("sleep %" PRIu64 " ms\n", @@ -1679,7 +1898,7 @@ static void *syncWriteInterlace(void *sarg) { } startTs = toolsGetTimestampUs(); - if (execInsert(pThreadInfo, generated)) { + if (execInsert(pThreadInfo, generated, &delay3)) { g_fail = true; goto free_of_interlace; } @@ -1733,7 +1952,8 @@ static void *syncWriteInterlace(void *sarg) { break; } - int64_t delay = endTs - startTs; + int64_t delay4 = endTs - startTs; + int64_t delay = delay1 + delay2 + delay3 + delay4; if (delay <=0) { debugPrint("thread[%d]: startTS: %"PRId64", endTS: %"PRId64"\n", pThreadInfo->threadID, startTs, endTs); @@ -1747,7 +1967,11 @@ static void *syncWriteInterlace(void *sarg) { tmfree(pdelay); } pThreadInfo->totalDelay += delay; + pThreadInfo->totalDelay1 += delay1; + pThreadInfo->totalDelay2 += delay2; + pThreadInfo->totalDelay3 += delay3; } + delay1 = delay2 = delay3 = 0; int64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30 * 1000) { @@ -1757,18 +1981,23 @@ static void *syncWriteInterlace(void *sarg) { pThreadInfo->threadID, pThreadInfo->totalInsertRows, (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); lastPrintTime = currentPrintTime; - lastTotalInsertRows = pThreadInfo->totalInsertRows; + lastTotalInsertRows = pThreadInfo->totalInsertRows; } } + free_of_interlace: cleanupAndPrint(pThreadInfo, "interlace"); + if(csvFile) { + fclose(csvFile); + } + tmfree(tagData); return NULL; } static int32_t prepareProgressDataStmt( threadInfo *pThreadInfo, SChildTable *childTbl, - int64_t *timestamp, uint64_t i, char *ttl) { + int64_t *timestamp, uint64_t i, char *ttl, int32_t *pkCur, int32_t *pkCnt, int64_t *delay1, int64_t *delay2, int64_t *delay3) { SSuperTable *stbInfo = pThreadInfo->stbInfo; char escapedTbName[TSDB_TABLE_NAME_LEN + 2] = "\0"; if (g_arguments->escape_character) { @@ -1778,6 +2007,7 @@ static int32_t prepareProgressDataStmt( snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", childTbl->name); } + int64_t start = toolsGetTimestampUs(); if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, escapedTbName)) { errorPrint( @@ -1786,13 +2016,15 @@ static int32_t prepareProgressDataStmt( taos_stmt_errstr(pThreadInfo->conn->stmt)); return -1; } + *delay1 = toolsGetTimestampUs() - start; + int32_t n =0; int32_t generated = bindParamBatch( pThreadInfo, (g_arguments->reqPerReq > (stbInfo->insertRows - i)) ? (stbInfo->insertRows - i) : g_arguments->reqPerReq, - *timestamp, childTbl); - *timestamp += generated * stbInfo->timestamp_step; + *timestamp, childTbl, pkCur, pkCnt, &n, delay2, delay3); + *timestamp += n * stbInfo->timestamp_step; return generated; } @@ -1818,7 +2050,7 @@ static void makeTimestampDisorder( static int32_t prepareProgressDataSmlJsonText( threadInfo *pThreadInfo, uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { + int64_t *timestamp, uint64_t i, char *ttl, int32_t *pkCur, int32_t *pkCnt) { // prepareProgressDataSmlJsonText SSuperTable *stbInfo = pThreadInfo->stbInfo; int32_t generated = 0; @@ -1879,7 +2111,12 @@ static int32_t prepareProgressDataSmlJsonText( if (pos >= g_arguments->prepared_rand) { pos = 0; } - *timestamp += stbInfo->timestamp_step; + + // primay key repeat ts count + if (!stbInfo->primary_key || needChangeTs(stbInfo, pkCur, pkCnt)) { + *timestamp += stbInfo->timestamp_step; + } + if (stbInfo->disorderRatio > 0) { makeTimestampDisorder(timestamp, stbInfo); } @@ -1902,7 +2139,7 @@ static int32_t prepareProgressDataSmlJsonText( static int32_t prepareProgressDataSmlJson( threadInfo *pThreadInfo, uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { + int64_t *timestamp, uint64_t i, char *ttl, int32_t *pkCur, int32_t *pkCnt) { // prepareProgressDataSmlJson SDataBase * database = pThreadInfo->dbInfo; SSuperTable *stbInfo = pThreadInfo->stbInfo; @@ -1932,7 +2169,12 @@ static int32_t prepareProgressDataSmlJson( if (pos >= g_arguments->prepared_rand) { pos = 0; } - *timestamp += stbInfo->timestamp_step; + + // primay key repeat ts count + if (!stbInfo->primary_key || needChangeTs(stbInfo, pkCur, pkCnt)) { + *timestamp += stbInfo->timestamp_step; + } + if (stbInfo->disorderRatio > 0) { makeTimestampDisorder(timestamp, stbInfo); } @@ -1955,7 +2197,7 @@ static int32_t prepareProgressDataSmlJson( static int32_t prepareProgressDataSmlLineOrTelnet( threadInfo *pThreadInfo, uint64_t tableSeq, char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, int protocol) { + int64_t *timestamp, uint64_t i, char *ttl, int protocol, int32_t *pkCur, int32_t *pkCnt) { // prepareProgressDataSmlLine SSuperTable *stbInfo = pThreadInfo->stbInfo; int32_t generated = 0; @@ -1987,7 +2229,11 @@ static int32_t prepareProgressDataSmlLineOrTelnet( if (pos >= g_arguments->prepared_rand) { pos = 0; } - *timestamp += stbInfo->timestamp_step; + // primay key repeat ts count + if (!stbInfo->primary_key || needChangeTs(stbInfo, pkCur, pkCnt)) { + *timestamp += stbInfo->timestamp_step; + } + if (stbInfo->disorderRatio > 0) { makeTimestampDisorder(timestamp, stbInfo); } @@ -2003,7 +2249,7 @@ static int32_t prepareProgressDataSml( threadInfo *pThreadInfo, SChildTable *childTbl, uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { + int64_t *timestamp, uint64_t i, char *ttl, int32_t *pkCur, int32_t *pkCnt) { // prepareProgressDataSml SSuperTable *stbInfo = pThreadInfo->stbInfo; @@ -2022,19 +2268,19 @@ static int32_t prepareProgressDataSml( pThreadInfo, tableSeq, sampleDataBuf, - timestamp, i, ttl, protocol); + timestamp, i, ttl, protocol, pkCur, pkCnt); break; case TSDB_SML_JSON_PROTOCOL: generated = prepareProgressDataSmlJsonText( pThreadInfo, tableSeq - pThreadInfo->start_table_from, - timestamp, i, ttl); + timestamp, i, ttl, pkCur, pkCnt); break; case SML_JSON_TAOS_FORMAT: generated = prepareProgressDataSmlJson( pThreadInfo, tableSeq, - timestamp, i, ttl); + timestamp, i, ttl, pkCur, pkCnt); break; default: errorPrint("%s() LN%d: unknown protcolor: %d\n", @@ -2045,12 +2291,47 @@ static int32_t prepareProgressDataSml( return generated; } +// if return true, timestmap must add timestap_step, else timestamp no need changed +bool needChangeTs(SSuperTable * stbInfo, int32_t *pkCur, int32_t *pkCnt) { + // check need generate cnt + if(*pkCnt == 0) { + if (stbInfo->repeat_ts_min >= stbInfo->repeat_ts_max) { + // fixed count value is max + if (stbInfo->repeat_ts_max == 0){ + return true; + } + + *pkCnt = stbInfo->repeat_ts_max; + } else { + // random range + *pkCnt = RD(stbInfo->repeat_ts_max + 1); + if(*pkCnt < stbInfo->repeat_ts_min) { + *pkCnt = (*pkCnt + stbInfo->repeat_ts_min) % stbInfo->repeat_ts_max; + } + } + } + + // compare with current value + *pkCur = *pkCur + 1; + if(*pkCur >= *pkCnt) { + // reset zero + *pkCur = 0; + *pkCnt = 0; + return true; + } else { + // add one + return false; + } +} + static int32_t prepareProgressDataSql( - threadInfo *pThreadInfo, - SChildTable *childTbl, uint64_t tableSeq, - char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, - int32_t *pos, uint64_t *len) { + threadInfo *pThreadInfo, + SChildTable *childTbl, + char* tagData, + uint64_t tableSeq, + char *sampleDataBuf, + int64_t *timestamp, uint64_t i, char *ttl, + int32_t *pos, uint64_t *len, int32_t* pkCur, int32_t* pkCnt) { // prepareProgressDataSql int32_t generated = 0; SDataBase *database = pThreadInfo->dbInfo; @@ -2068,7 +2349,7 @@ static int32_t prepareProgressDataSql( STR_INSERT_INTO, database->dbName, childTbl->name, database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + + tagData + stbInfo->lenOfTags * tableSeq, ttl); } else { *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, @@ -2089,7 +2370,7 @@ static int32_t prepareProgressDataSql( childTbl->name, stbInfo->partialColNameBuf, database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + + tagData + stbInfo->lenOfTags * tableSeq, ttl); } else { *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, @@ -2131,7 +2412,11 @@ static int32_t prepareProgressDataSql( if (*pos >= g_arguments->prepared_rand) { *pos = 0; } - *timestamp += stbInfo->timestamp_step; + // primary key + if(!stbInfo->primary_key || needChangeTs(stbInfo, pkCur, pkCnt)) { + *timestamp += stbInfo->timestamp_step; + } + generated++; if (*len > (TSDB_MAX_ALLOWED_SQL_LEN - stbInfo->lenOfCols)) { @@ -2150,55 +2435,48 @@ void *syncWriteProgressive(void *sarg) { SDataBase * database = pThreadInfo->dbInfo; SSuperTable *stbInfo = pThreadInfo->stbInfo; + loadChildTableInfo(pThreadInfo); + // special deal flow for TAOSC_IFACE if (insertDataMix(pThreadInfo, database, stbInfo)) { // request be dealt by this function , so return return NULL; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - if (0 == pThreadInfo->vg->tbCountPerVgId) { - return NULL; - } - } else { - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); - } -#else infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); -#endif + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); + uint64_t lastPrintTime = toolsGetTimestampMs(); uint64_t lastTotalInsertRows = 0; int64_t startTs = toolsGetTimestampUs(); int64_t endTs; + FILE* csvFile = NULL; + char* tagData = NULL; + bool stmt = stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating; + bool smart = SMART_IF_FAILED == stbInfo->continueIfFail; + bool acreate = (stbInfo->iface == TAOSC_IFACE || stbInfo->iface == REST_IFACE) && stbInfo->autoTblCreating; + int w = 0; + if (stmt || smart || acreate) { + csvFile = openTagCsv(stbInfo); + tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); + } + for (uint64_t tableSeq = pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; tableSeq++) { char *sampleDataBuf; SChildTable *childTbl; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { + + if (g_arguments->bind_vgroup) { childTbl = pThreadInfo->vg->childTblArray[tableSeq]; } else { - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; + childTbl = stbInfo->childTblArray[tableSeq]; } -#else - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; -#endif + debugPrint("tableSeq=%"PRId64" childTbl->name=%s\n", tableSeq, childTbl->name); + if (childTbl->useOwnSample) { sampleDataBuf = childTbl->sampleDataBuf; } else { @@ -2208,22 +2486,56 @@ void *syncWriteProgressive(void *sarg) { int64_t timestamp = pThreadInfo->start_time; uint64_t len = 0; int32_t pos = 0; - if (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating) { + int32_t pkCur = 0; // record generate same timestamp current count + int32_t pkCnt = 0; // record generate same timestamp count + int64_t delay1 = 0; + int64_t delay2 = 0; + int64_t delay3 = 0; + if (stmt) { taos_stmt_close(pThreadInfo->conn->stmt); - pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + if(stbInfo->autoTblCreating) { + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + } else { + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + pThreadInfo->conn->stmt = taos_stmt_init_with_options(pThreadInfo->conn->taos, &op); + } + if (NULL == pThreadInfo->conn->stmt) { errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); g_fail = true; goto free_of_progressive; } + } - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tableSeq)) { + if(stmt || smart || acreate) { + // generator + if (w == 0) { + if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { + g_fail = true; + goto free_of_progressive; + } + } + } + + if (stmt) { + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tagData, w)) { g_fail = true; goto free_of_progressive; } } + if(stmt || smart || acreate) { + // move next + if (++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } + } + char ttl[SMALL_BUFF_LEN] = ""; if (stbInfo->ttl != 0) { snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); @@ -2239,14 +2551,15 @@ void *syncWriteProgressive(void *sarg) { generated = prepareProgressDataSql( pThreadInfo, childTbl, - tableSeq, + tagData, + w, sampleDataBuf, - ×tamp, i, ttl, &pos, &len); + ×tamp, i, ttl, &pos, &len, &pkCur, &pkCnt); break; case STMT_IFACE: { generated = prepareProgressDataStmt( pThreadInfo, - childTbl, ×tamp, i, ttl); + childTbl, ×tamp, i, ttl, &pkCur, &pkCnt, &delay1, &delay2, &delay3); break; } case SML_REST_IFACE: @@ -2254,7 +2567,7 @@ void *syncWriteProgressive(void *sarg) { generated = prepareProgressDataSml( pThreadInfo, childTbl, - tableSeq, ×tamp, i, ttl); + tableSeq, ×tamp, i, ttl, &pkCur, &pkCnt); break; default: break; @@ -2268,7 +2581,7 @@ void *syncWriteProgressive(void *sarg) { } // only measure insert startTs = toolsGetTimestampUs(); - int code = execInsert(pThreadInfo, generated); + int code = execInsert(pThreadInfo, generated, &delay3); if (code) { if (NO_IF_FAILED == stbInfo->continueIfFail) { warnPrint("The super table parameter " @@ -2281,20 +2594,35 @@ void *syncWriteProgressive(void *sarg) { "continueIfFail: %d, " "will continue to insert ..\n", stbInfo->continueIfFail); - } else if (SMART_IF_FAILED == stbInfo->continueIfFail) { + } else if (smart) { warnPrint("The super table parameter " "continueIfFail: %d, will create table " "then insert ..\n", stbInfo->continueIfFail); + + // generator + if (w == 0) { + if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { + g_fail = true; + goto free_of_progressive; + } + } + int ret = smartContinueIfFail( pThreadInfo, - childTbl, i, ttl); + childTbl, tagData, w, ttl); if (0 != ret) { g_fail = true; goto free_of_progressive; } - code = execInsert(pThreadInfo, generated); + // move next + if (++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } + + code = execInsert(pThreadInfo, generated, &delay3); if (code) { g_fail = true; goto free_of_progressive; @@ -2317,6 +2645,18 @@ void *syncWriteProgressive(void *sarg) { toolsMsleep((int32_t)stbInfo->insert_interval); } + // flush + if (database->flush) { + char sql[260] = ""; + sprintf(sql, "flush database %s", database->dbName); + code = executeSql(pThreadInfo->conn->taos,sql); + if (code != 0) { + perfPrint(" %s failed. error code = 0x%x\n", sql, code); + } else { + perfPrint(" %s ok.\n", sql); + } + } + pThreadInfo->totalInsertRows += generated; if (g_arguments->terminate) { @@ -2359,7 +2699,8 @@ void *syncWriteProgressive(void *sarg) { break; } - int64_t delay = endTs - startTs; + int64_t delay4 = endTs - startTs; + int64_t delay = delay1 + delay2 + delay3 + delay4; if (delay <= 0) { debugPrint("thread[%d]: startTs: %"PRId64", endTs: %"PRId64"\n", pThreadInfo->threadID, startTs, endTs); @@ -2373,7 +2714,11 @@ void *syncWriteProgressive(void *sarg) { tmfree(pDelay); } pThreadInfo->totalDelay += delay; + pThreadInfo->totalDelay1 += delay1; + pThreadInfo->totalDelay2 += delay2; + pThreadInfo->totalDelay3 += delay3; } + delay1 = delay2 = delay3 = 0; int64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30 * 1000) { @@ -2392,6 +2737,10 @@ void *syncWriteProgressive(void *sarg) { } // tableSeq free_of_progressive: cleanupAndPrint(pThreadInfo, "progressive"); + if(csvFile) { + fclose(csvFile); + } + tmfree(tagData); return NULL; } @@ -2638,10 +2987,12 @@ static int parseBufferToStmtBatch(SSuperTable* stbInfo) { static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - snprintf(stbInfo->childTblArray[i]->name, + char childName[TSDB_TABLE_NAME_LEN]={0}; + snprintf(childName, TSDB_TABLE_NAME_LEN, "%s%" PRIu64 "", stbInfo->childTblPrefix, i); + stbInfo->childTblArray[i]->name = strdup(childName); debugPrint("%s(): %s\n", __func__, stbInfo->childTblArray[i]->name); } @@ -2652,10 +3003,12 @@ static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { static int64_t fillChildTblNameByFromTo(SDataBase *database, SSuperTable* stbInfo) { for (int64_t i = stbInfo->childTblFrom; i < stbInfo->childTblTo; i++) { - snprintf(stbInfo->childTblArray[i-stbInfo->childTblFrom]->name, + char childName[TSDB_TABLE_NAME_LEN]={0}; + snprintf(childName, TSDB_TABLE_NAME_LEN, "%s%" PRIu64 "", stbInfo->childTblPrefix, i); + stbInfo->childTblArray[i-stbInfo->childTblFrom]->name = strdup(childName); } return (stbInfo->childTblTo-stbInfo->childTblFrom); @@ -2693,8 +3046,10 @@ static int64_t fillChildTblNameByLimitOffset(SDataBase *database, TAOS_ROW row = NULL; while ((row = taos_fetch_row(res)) != NULL) { int *lengths = taos_fetch_lengths(res); - strncpy(stbInfo->childTblArray[count]->name, row[0], lengths[0]); - stbInfo->childTblArray[count]->name[lengths[0] + 1] = '\0'; + char * childName = benchCalloc(1, lengths[0] + 1, true); + strncpy(childName, row[0], lengths[0]); + childName[lengths[0]] = '\0'; + stbInfo->childTblArray[count]->name = childName; debugPrint("stbInfo->childTblArray[%" PRId64 "]->name: %s\n", count, stbInfo->childTblArray[count]->name); count++; @@ -2743,15 +3098,32 @@ static void preProcessArgument(SSuperTable *stbInfo) { static int printTotalDelay(SDataBase *database, int64_t totalDelay, + int64_t totalDelay1, + int64_t totalDelay2, + int64_t totalDelay3, BArray *total_delay_list, int threads, int64_t totalInsertRows, - int64_t start, int64_t end) { - succPrint("Spent %.6f seconds to insert rows: %" PRIu64 - " with %d thread(s) into %s %.2f records/second\n", - (end - start)/1E6, totalInsertRows, threads, + int64_t spend) { + // zero check + if (total_delay_list->size == 0 || spend == 0 || threads == 0) { + return -1; + } + + char subDelay[128] = ""; + if(totalDelay1 + totalDelay2 + totalDelay3 > 0) { + sprintf(subDelay, " stmt delay1=%.2fs delay2=%.2fs delay3=%.2fs", + totalDelay1/threads/1E6, + totalDelay2/threads/1E6, + totalDelay3/threads/1E6); + } + + succPrint("Spent %.6f (real %.6f) seconds to insert rows: %" PRIu64 + " with %d thread(s) into %s %.2f (real %.2f) records/second%s\n", + spend/1E6, totalDelay/threads/1E6, totalInsertRows, threads, database->dbName, - (double)(totalInsertRows / ((end - start)/1E6))); + (double)(totalInsertRows / (spend/1E6)), + (double)(totalInsertRows / (totalDelay/threads/1E6)), subDelay); if (!total_delay_list->size) { return -1; } @@ -2803,8 +3175,10 @@ static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { if (stbInfo->childTblCount == 1 && stbInfo->tags->size == 0) { // Normal table - snprintf(stbInfo->childTblArray[0]->name, TSDB_TABLE_NAME_LEN, + char childName[TSDB_TABLE_NAME_LEN]={0}; + snprintf(childName, TSDB_TABLE_NAME_LEN, "%s", stbInfo->stbName); + stbInfo->childTblArray[0]->name = strdup(childName); } else if ((stbInfo->iface != SML_IFACE && stbInfo->iface != SML_REST_IFACE) && stbInfo->childTblExists) { @@ -2816,16 +3190,84 @@ static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { return ntables; } -static int startMultiThreadInsertData(SDataBase* database, - SSuperTable* stbInfo) { - if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) - && !stbInfo->use_metric) { - errorPrint("%s", "schemaless cannot work without stable\n"); - return -1; +// last ts fill to filllBackTime +static bool fillSTableLastTs(SDataBase *database, SSuperTable *stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return false; } + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "select last(ts) from %s.`%s`", database->dbName, stbInfo->stbName); - preProcessArgument(stbInfo); + infoPrint("fillBackTime: %s\n", cmd); + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return false; + } + + TAOS_ROW row = taos_fetch_row(res); + if(row == NULL) { + taos_free_result(res); + closeBenchConn(conn); + return false; + } + + char lastTs[128]; + memset(lastTs, 0, sizeof(lastTs)); + + stbInfo->startFillbackTime = *(int64_t*)row[0]; + toolsFormatTimestamp(lastTs, stbInfo->startFillbackTime, database->precision); + infoPrint("fillBackTime: get ok %s.%s last ts=%s \n", database->dbName, stbInfo->stbName, lastTs); + + taos_free_result(res); + closeBenchConn(conn); + + return true; +} + +// calcNow expression fill to timestamp_start +static bool calcExprFromServer(SDataBase *database, SSuperTable *stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return false; + } + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "select %s", stbInfo->calcNow); + + infoPrint("calcExprFromServer: %s\n", cmd); + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return false; + } + + TAOS_ROW row = taos_fetch_row(res); + if(row == NULL) { + taos_free_result(res); + closeBenchConn(conn); + return false; + } + + char ts[128]; + memset(ts, 0, sizeof(ts)); + + stbInfo->startTimestamp = *(int64_t*)row[0]; + toolsFormatTimestamp(ts, stbInfo->startTimestamp, database->precision); + infoPrint("calcExprFromServer: get ok. %s = %s \n", stbInfo->calcNow, ts); + + taos_free_result(res); + closeBenchConn(conn); + + return true; +} +int64_t obtainTableCount(SDataBase* database, SSuperTable* stbInfo) { + // ntable calc int64_t ntables; if (stbInfo->childTblTo > 0) { ntables = stbInfo->childTblTo - stbInfo->childTblFrom; @@ -2834,202 +3276,200 @@ static int startMultiThreadInsertData(SDataBase* database, } else { ntables = stbInfo->childTblCount; } - if (ntables == 0) { + + return ntables; +} + +// assign table to thread with vgroups, return assign thread count +int32_t assignTableToThread(SDataBase* database, SSuperTable* stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { return 0; } + int32_t threads = 0; - uint64_t tableFrom = 0; - int32_t threads = g_arguments->nthreads; - int64_t a = 0, b = 0; - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { + // calc table count per vgroup + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + closeBenchConn(conn); return -1; } - - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->tbCountPerVgId++; - } - } - } - - threads = 0; - for (int v = 0; v < database->vgroups; v++) { + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { SVGroup *vg = benchArrayGet(database->vgArray, v); - infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", - vg->tbCountPerVgId, database->dbName, v, vg->vgId); - if (vg->tbCountPerVgId) { - threads++; - } else { - continue; + if (vgId == vg->vgId) { + vg->tbCountPerVgId++; } - vg->childTblArray = benchCalloc( - vg->tbCountPerVgId, sizeof(SChildTable*), true); - vg->tbOffset = 0; } - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); + } - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->childTblArray[vg->tbOffset] = - stbInfo->childTblArray[i]; - vg->tbOffset++; - } - } + // malloc vg->childTblArray memory with table count + for (int v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + infoPrint("Total %"PRId64" tables on %s's vgroup %d (id: %d)\n", + vg->tbCountPerVgId, database->dbName, v, vg->vgId); + if (vg->tbCountPerVgId) { + threads++; + } else { + continue; } - closeBenchConn(conn); - } else { - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; + vg->childTblArray = benchCalloc(vg->tbCountPerVgId, sizeof(SChildTable*), true); + vg->tbOffset = 0; + } + + // set vg->childTblArray data + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + + closeBenchConn(conn); + return 0; } - b = 0; - if (threads != 0) { - b = ntables % threads; + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->childTblArray[vg->tbOffset] = stbInfo->childTblArray[i]; + vg->tbOffset++; + } } } + closeBenchConn(conn); - int32_t vgFrom = 0; -#else - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); + return threads; +} - for (int32_t i = 0; i < threads; i++) { +// init insert thread +int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, int64_t div, int64_t mod) { + int32_t ret = -1; + uint64_t tbNext = stbInfo->childTblFrom; + int32_t vgNext = 0; + FILE* csvFile = NULL; + char* tagData = NULL; + bool stmtN = stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating == false; + int w = 0; + + if (stmtN) { + csvFile = openTagCsv(stbInfo); + tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); + } + + for (int32_t i = 0; i < nthreads; i++) { + // set table threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->dbInfo = database; - pThreadInfo->stbInfo = stbInfo; + pThreadInfo->threadID = i; + pThreadInfo->dbInfo = database; + pThreadInfo->stbInfo = stbInfo; pThreadInfo->start_time = stbInfo->startTimestamp; + pThreadInfo->pos = 0; + pThreadInfo->samplePos = 0; pThreadInfo->totalInsertRows = 0; - pThreadInfo->samplePos = 0; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - int32_t j; - for (j = vgFrom; i < database->vgroups; j++) { + + if (g_arguments->bind_vgroup) { + for (int32_t j = vgNext; j < database->vgroups; j++) { SVGroup *vg = benchArrayGet(database->vgArray, j); if (0 == vg->tbCountPerVgId) { continue; } - pThreadInfo->vg = vg; + pThreadInfo->vg = vg; + pThreadInfo->ntables = vg->tbCountPerVgId; pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = vg->tbCountPerVgId; - pThreadInfo->end_table_to = vg->tbCountPerVgId-1; + pThreadInfo->end_table_to = vg->tbCountPerVgId - 1; + vgNext = j + 1; break; - } - vgFrom = j + 1; + } } else { - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->start_table_from = tbNext; + pThreadInfo->ntables = i < mod ? div + 1 : div; + pThreadInfo->end_table_to = i < mod ? tbNext + div : tbNext + div - 1; + tbNext = pThreadInfo->end_table_to + 1; } -#else - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; -#endif // TD_VER_COMPATIBLE_3_0_0_0 + + // init conn pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); switch (stbInfo->iface) { + // rest case REST_IFACE: { if (stbInfo->interlaceRows > 0) { pThreadInfo->buffer = new_ds(0); } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); } int sockfd = createSockFd(); if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); + goto END; } pThreadInfo->sockfd = sockfd; break; } + // stmt case STMT_IFACE: { pThreadInfo->conn = initBenchConn(); if (NULL == pThreadInfo->conn) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); + goto END; + } + taos_stmt_close(pThreadInfo->conn->stmt); + if(stbInfo->autoTblCreating) { + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + } else { + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + pThreadInfo->conn->stmt = taos_stmt_init_with_options(pThreadInfo->conn->taos, &op); } - pThreadInfo->conn->stmt = - taos_stmt_init(pThreadInfo->conn->taos); if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - FREE_RESOURCE(); - return -1; + errorPrint("taos_stmt_init() failed, reason: %s\n", taos_errstr(NULL)); + goto END; } if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } - if (!stbInfo->autoTblCreating) { - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, 0)) { - FREE_RESOURCE(); - return -1; + if (stmtN) { + // generator + if (w == 0) { + if(!generateTagData(stbInfo, tagData, TAG_BATCH_COUNT, csvFile)) { + goto END; + } } + + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tagData, w)) { + goto END; + } + + // move next + if (++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } } - pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); - pThreadInfo->bind_ts_array = - benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, - true); - pThreadInfo->bindParams = benchCalloc( - 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), - true); - pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, - true); + // malloc bind + pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); + pThreadInfo->bind_ts_array = benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, true); + pThreadInfo->bindParams = benchCalloc(1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), true); + pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, true); + parseBufferToStmtBatch(stbInfo); - for (int64_t child = 0; - child < stbInfo->childTblCount; child++) { + for (int64_t child = 0; child < stbInfo->childTblCount; child++) { SChildTable *childTbl = stbInfo->childTblArray[child]; if (childTbl->useOwnSample) { parseBufferToStmtBatchChildTbl(stbInfo, childTbl); @@ -3038,44 +3478,34 @@ static int startMultiThreadInsertData(SDataBase* database, break; } + // sml rest case SML_REST_IFACE: { int sockfd = createSockFd(); if (sockfd < 0) { - free(pids); - free(infos); - return -1; + goto END; } pThreadInfo->sockfd = sockfd; } + // sml case SML_IFACE: { pThreadInfo->conn = initBenchConn(); if (pThreadInfo->conn == NULL) { errorPrint("%s() init connection failed\n", __func__); - FREE_RESOURCE(); - return -1; + goto END; } if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } - pThreadInfo->max_sql_len = - stbInfo->lenOfCols + stbInfo->lenOfTags; + pThreadInfo->max_sql_len = stbInfo->lenOfCols + stbInfo->lenOfTags; if (stbInfo->iface == SML_REST_IFACE) { - pThreadInfo->buffer = - benchCalloc(1, g_arguments->reqPerReq * - (1 + pThreadInfo->max_sql_len), true); + pThreadInfo->buffer = benchCalloc(1, g_arguments->reqPerReq * (1 + pThreadInfo->max_sql_len), true); } int protocol = stbInfo->lineProtocol; - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - pThreadInfo->sml_tags = - (char **)benchCalloc(pThreadInfo->ntables, - sizeof(char *), true); + if (TSDB_SML_JSON_PROTOCOL != protocol && SML_JSON_TAOS_FORMAT != protocol) { + pThreadInfo->sml_tags = (char **)benchCalloc(pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { - pThreadInfo->sml_tags[t] = - benchCalloc(1, stbInfo->lenOfTags, true); + pThreadInfo->sml_tags[t] = benchCalloc(1, stbInfo->lenOfTags, true); } for (int t = 0; t < pThreadInfo->ntables; t++) { @@ -3089,20 +3519,14 @@ static int startMultiThreadInsertData(SDataBase* database, debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, pThreadInfo->sml_tags[t]); } - pThreadInfo->lines = - benchCalloc(g_arguments->reqPerReq, - sizeof(char *), true); - - for (int j = 0; (j < g_arguments->reqPerReq - && !g_arguments->terminate); j++) { - pThreadInfo->lines[j] = - benchCalloc(1, pThreadInfo->max_sql_len, true); + pThreadInfo->lines = benchCalloc(g_arguments->reqPerReq, sizeof(char *), true); + for (int j = 0; (j < g_arguments->reqPerReq && !g_arguments->terminate); j++) { + pThreadInfo->lines[j] = benchCalloc(1, pThreadInfo->max_sql_len, true); } } else { - pThreadInfo->json_array = tools_cJSON_CreateArray(); - pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); - pThreadInfo->sml_tags_json_array = (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); + pThreadInfo->json_array = tools_cJSON_CreateArray(); + pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); + pThreadInfo->sml_tags_json_array = (char **)benchCalloc( pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { generateSmlJsonTags( @@ -3116,48 +3540,33 @@ static int startMultiThreadInsertData(SDataBase* database, pThreadInfo->start_table_from, t); } } - pThreadInfo->lines = (char **)benchCalloc( - 1, sizeof(char *), true); - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - pThreadInfo->line_buf_len = - g_arguments->reqPerReq * - accumulateRowLen(pThreadInfo->stbInfo->tags, - pThreadInfo->stbInfo->iface); - debugPrint("%s() LN%d, line_buf_len=%d\n", - __func__, __LINE__, pThreadInfo->line_buf_len); - pThreadInfo->lines[0] = benchCalloc( - 1, pThreadInfo->line_buf_len, true); - pThreadInfo->sml_json_value_array = - (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); + pThreadInfo->lines = (char **)benchCalloc(1, sizeof(char *), true); + if (0 == stbInfo->interlaceRows && TSDB_SML_JSON_PROTOCOL == protocol) { + pThreadInfo->line_buf_len = g_arguments->reqPerReq * accumulateRowLen(pThreadInfo->stbInfo->tags, pThreadInfo->stbInfo->iface); + debugPrint("%s() LN%d, line_buf_len=%d\n", __func__, __LINE__, pThreadInfo->line_buf_len); + pThreadInfo->lines[0] = benchCalloc(1, pThreadInfo->line_buf_len, true); + pThreadInfo->sml_json_value_array = (char **)benchCalloc(pThreadInfo->ntables, sizeof(char *), true); for (int t = 0; t < pThreadInfo->ntables; t++) { - generateSmlJsonValues( - pThreadInfo->sml_json_value_array, stbInfo, t); + generateSmlJsonValues(pThreadInfo->sml_json_value_array, stbInfo, t); } } } break; } + // taos case TAOSC_IFACE: { pThreadInfo->conn = initBenchConn(); if (pThreadInfo->conn == NULL) { errorPrint("%s() failed to connect\n", __func__); - FREE_RESOURCE(); - return -1; + goto END; } char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", + g_arguments->escape_character ? "USE `%s`" : "USE %s", database->dbName); if (queryDbExecCall(pThreadInfo->conn, command)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - tmfree(command); - return -1; + errorPrint("taos select database(%s) failed\n", database->dbName); + goto END; } tmfree(command); command = NULL; @@ -3165,11 +3574,9 @@ static int startMultiThreadInsertData(SDataBase* database, if (stbInfo->interlaceRows > 0) { pThreadInfo->buffer = new_ds(0); } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); if (g_arguments->check_sql) { - pThreadInfo->csql = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + pThreadInfo->csql = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); } } @@ -3181,25 +3588,92 @@ static int startMultiThreadInsertData(SDataBase* database, } } - infoPrint("Estimate memory usage: %.2fMB\n", - (double)g_memoryUsage / 1048576); - prompt(0); + // success + ret = 0; + +END: + if (csvFile) { + fclose(csvFile); + } + tmfree(tagData); + return ret; +} + +#define EMPTY_SLOT -1 +// run with limit thread +int32_t runInsertLimitThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, int32_t limitThread, threadInfo *infos, pthread_t *pids) { + infoPrint("run with bind vgroups limit thread. limit threads=%d nthread=%d\n", limitThread, nthreads); + + // slots save threadInfo array index + int32_t* slot = benchCalloc(limitThread, sizeof(int32_t), false); + int32_t t = 0; // thread index + for (int32_t i = 0; i < limitThread; i++) { + slot[i] = EMPTY_SLOT; + } + + while (!g_arguments->terminate) { + int32_t emptySlot = 0; + for (int32_t i = 0; i < limitThread; i++) { + int32_t idx = slot[i]; + // check slot thread end + if(idx != EMPTY_SLOT) { + if (pthread_tryjoin_np(pids[idx], NULL) == EBUSY ) { + // thread is running + toolsMsleep(2000); + } else { + // thread is end , set slot empty + infoPrint("slot[%d] finished tidx=%d. completed thread count=%d\n", i, slot[i], t); + slot[i] = EMPTY_SLOT; + } + } + + if (slot[i] == EMPTY_SLOT && t < nthreads) { + // slot is empty , set new thread to running + threadInfo *pThreadInfo = infos + t; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + t, NULL, syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + t, NULL, syncWriteProgressive, pThreadInfo); + } + + // save current and move next + slot[i] = t; + t++; + infoPrint("slot[%d] start new thread tidx=%d. \n", i, slot[i]); + } + + // check slot empty + if(slot[i] == EMPTY_SLOT) { + emptySlot++; + } + } + // check all thread end + if(emptySlot == limitThread) { + debugPrint("all threads(%d) is run finished.\n", nthreads); + break; + } else { + debugPrint("current thread index=%d all thread=%d\n", t, nthreads); + } + } + + return 0; +} + +// run +int32_t runInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids) { + infoPrint("run insert thread. real nthread=%d\n", nthreads); // create threads int threadCnt = 0; - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + for (int i = 0; i < nthreads && !g_arguments->terminate; i++) { threadInfo *pThreadInfo = infos + i; if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); + pthread_create(pids + i, NULL, syncWriteInterlace, pThreadInfo); } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); + pthread_create(pids + i, NULL, syncWriteProgressive, pThreadInfo); } threadCnt ++; - } - - int64_t start = toolsGetTimestampUs(); + } // wait threads for (int i = 0; i < threadCnt; i++) { @@ -3207,16 +3681,24 @@ static int startMultiThreadInsertData(SDataBase* database, pthread_join(pids[i], NULL); } - int64_t end = toolsGetTimestampUs()+1; + return 0; +} + + +// exit and free resource +int32_t exitInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthreads, threadInfo *infos, pthread_t *pids, int64_t spend) { if (g_arguments->terminate) toolsMsleep(100); BArray * total_delay_list = benchArrayInit(1, sizeof(int64_t)); int64_t totalDelay = 0; + int64_t totalDelay1 = 0; + int64_t totalDelay2 = 0; + int64_t totalDelay3 = 0; uint64_t totalInsertRows = 0; // free threads resource - for (int i = 0; i < threads; i++) { + for (int i = 0; i < nthreads; i++) { threadInfo *pThreadInfo = infos + i; // free check sql if (pThreadInfo->csql) { @@ -3224,6 +3706,7 @@ static int startMultiThreadInsertData(SDataBase* database, pThreadInfo->csql = NULL; } + // close conn int protocol = stbInfo->lineProtocol; switch (stbInfo->iface) { case REST_IFACE: @@ -3284,6 +3767,12 @@ static int startMultiThreadInsertData(SDataBase* database, case STMT_IFACE: taos_stmt_close(pThreadInfo->conn->stmt); + + // free length + for (int c = 0; c < stbInfo->cols->size + 1; c++) { + TAOS_MULTI_BIND *param = (TAOS_MULTI_BIND *)(pThreadInfo->bindParams + sizeof(TAOS_MULTI_BIND) * c); + tmfree(param->length); + } tmfree(pThreadInfo->bind_ts); tmfree(pThreadInfo->bind_ts_array); tmfree(pThreadInfo->bindParams); @@ -3304,6 +3793,9 @@ static int startMultiThreadInsertData(SDataBase* database, } totalInsertRows += pThreadInfo->totalInsertRows; totalDelay += pThreadInfo->totalDelay; + totalDelay1 += pThreadInfo->totalDelay1; + totalDelay2 += pThreadInfo->totalDelay2; + totalDelay3 += pThreadInfo->totalDelay3; benchArrayAddBatch(total_delay_list, pThreadInfo->delayList->pData, pThreadInfo->delayList->size); tmfree(pThreadInfo->delayList); @@ -3313,6 +3805,8 @@ static int startMultiThreadInsertData(SDataBase* database, closeBenchConn(pThreadInfo->conn); pThreadInfo->conn = NULL; } + + } // calculate result @@ -3321,19 +3815,101 @@ static int startMultiThreadInsertData(SDataBase* database, if (g_arguments->terminate) toolsMsleep(100); - free(pids); - free(infos); + tmfree(pids); + tmfree(infos); - int ret = printTotalDelay(database, totalDelay, - total_delay_list, threads, - totalInsertRows, start, end); + // print result + int ret = printTotalDelay(database, totalDelay, totalDelay1, totalDelay2, totalDelay3, + total_delay_list, nthreads, totalInsertRows, spend); benchArrayDestroy(total_delay_list); - if (g_fail || ret) { + if (g_fail || ret != 0) { return -1; } return 0; } +static int startMultiThreadInsertData(SDataBase* database, SSuperTable* stbInfo) { + if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) + && !stbInfo->use_metric) { + errorPrint("%s", "schemaless cannot work without stable\n"); + return -1; + } + + // check argument valid + preProcessArgument(stbInfo); + + // ntable + int64_t ntables = obtainTableCount(database, stbInfo); + if (ntables == 0) { + errorPrint("insert table count is zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + + // assign table to thread + int32_t nthreads = g_arguments->nthreads; + int64_t div = 0; // ntable / nthread division + int64_t mod = 0; // ntable % nthread + int64_t spend = 0; + + if (g_arguments->bind_vgroup) { + nthreads = assignTableToThread(database, stbInfo); + if(nthreads == 0) { + errorPrint("bind vgroup assign theads count is zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + } else { + if(nthreads == 0) { + errorPrint("argument thread_count can not be zero. %s.%s\n", database->dbName, stbInfo->stbName); + return -1; + } + div = ntables / nthreads; + if (div < 1) { + nthreads = (int32_t)ntables; + div = 1; + } + mod = ntables % nthreads; + } + + + // init each thread information + pthread_t *pids = benchCalloc(1, nthreads * sizeof(pthread_t), true); + threadInfo *infos = benchCalloc(1, nthreads * sizeof(threadInfo), true); + + // init + int32_t ret = initInsertThread(database, stbInfo, nthreads, infos, div, mod); + if( ret != 0) { + errorPrint("init insert thread failed. %s.%s\n", database->dbName, stbInfo->stbName); + tmfree(pids); + tmfree(infos); + return ret; + } + + infoPrint("Estimate memory usage: %.2fMB\n", (double)g_memoryUsage / 1048576); + prompt(0); + + + // run + int64_t start = toolsGetTimestampUs(); + if(g_arguments->bind_vgroup && g_arguments->nthreads < nthreads ) { + // need many batch execute all threads + ret = runInsertLimitThread(database, stbInfo, nthreads, g_arguments->nthreads, infos, pids); + } else { + // only one batch execute all threads + ret = runInsertThread(database, stbInfo, nthreads, infos, pids); + } + + int64_t end = toolsGetTimestampUs(); + if(end == start) { + spend = 1; + } else { + spend = end - start; + } + + // exit + ret = exitInsertThread(database, stbInfo, nthreads, infos, pids, spend); + return ret; +} + static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { int rows = 0; char command[SHORT_1K_SQL_BUFF_LEN]; @@ -3496,6 +4072,7 @@ int insertTestProcess() { prompt(0); encodeAuthBase64(); + //loop create database for (int i = 0; i < g_arguments->databases->size; i++) { if (REST_IFACE == g_arguments->iface) { if (0 != convertServAddr(g_arguments->iface, @@ -3523,8 +4100,25 @@ int insertTestProcess() { return -1; } succPrint("created database (%s)\n", database->dbName); + } else { + // database already exist, get vgroups from server + if (database->superTbls) { + SBenchConn* conn = initBenchConn(); + if (conn) { + int32_t vgroups = getVgroupsOfDb(conn, database); + if (vgroups <=0) { + closeBenchConn(conn); + errorPrint("Database %s's vgroups is zero.\n", database->dbName); + return -1; + } + closeBenchConn(conn); + succPrint("Database (%s) get vgroups num is %d from server.\n", database->dbName, vgroups); + } + } } } + + // create super table && fill child tables && prepareSampleData for (int i = 0; i < g_arguments->databases->size; i++) { SDataBase * database = benchArrayGet(g_arguments->databases, i); if (database->superTbls) { @@ -3544,6 +4138,16 @@ int insertTestProcess() { } } } + // fill last ts from super table + if(stbInfo->autoFillback && stbInfo->childTblExists) { + fillSTableLastTs(database, stbInfo); + } + + // calc now + if(stbInfo->calcNow) { + calcExprFromServer(database, stbInfo); + } + // check fill child table count valid if(fillChildTblName(database, stbInfo) <= 0) { infoPrint(" warning fill childs table count is zero, please check parameters in json is correct. database:%s stb: %s \n", database->dbName, stbInfo->stbName); @@ -3551,10 +4155,20 @@ int insertTestProcess() { if (0 != prepareSampleData(database, stbInfo)) { return -1; } + + // execute sqls + if (stbInfo->sqls) { + char **sqls = stbInfo->sqls; + while (*sqls) { + queryDbExec(database, stbInfo, *sqls); + sqls++; + } + } } } } + // tsma if (g_arguments->taosc_version == 3) { for (int i = 0; i < g_arguments->databases->size; i++) { SDataBase* database = benchArrayGet(g_arguments->databases, i); diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 55d74dd3..a137f90a 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -177,7 +177,7 @@ char* genBatColsNames(threadInfo* info, SSuperTable* stb) { // // generate head // -uint32_t genInsertPreSql(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tableName, uint64_t tableSeq, char* pstr) { +uint32_t genInsertPreSql(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tableName, char* tagData, uint64_t tableSeq, char* pstr) { uint32_t len = 0; if (stb->genRowRule == RULE_OLD || stb->genRowRule == RULE_MIX_RANDOM) { @@ -190,7 +190,7 @@ uint32_t genInsertPreSql(threadInfo* info, SDataBase* db, SSuperTable* stb, char if (stb->partialColNum == stb->cols->size) { if (stb->autoTblCreating) { len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, "%s %s.%s USING %s.%s TAGS (%s) %s VALUES ", STR_INSERT_INTO, db->dbName, - tableName, db->dbName, stb->stbName, stb->tagDataBuf + stb->lenOfTags * tableSeq, ttl); + tableName, db->dbName, stb->stbName, tagData + stb->lenOfTags * tableSeq, ttl); } else { len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, "%s %s.%s VALUES ", STR_INSERT_INTO, db->dbName, tableName); } @@ -198,7 +198,7 @@ uint32_t genInsertPreSql(threadInfo* info, SDataBase* db, SSuperTable* stb, char if (stb->autoTblCreating) { len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, "%s %s.%s (%s) USING %s.%s TAGS (%s) %s VALUES ", STR_INSERT_INTO, db->dbName, tableName, stb->partialColNameBuf, db->dbName, stb->stbName, - stb->tagDataBuf + stb->lenOfTags * tableSeq, ttl); + tagData + stb->lenOfTags * tableSeq, ttl); } else { len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, "%s %s.%s (%s) VALUES ", STR_INSERT_INTO, db->dbName, tableName, stb->partialColNameBuf); @@ -219,10 +219,19 @@ uint32_t genInsertPreSql(threadInfo* info, SDataBase* db, SSuperTable* stb, char // new mix rule int32_t max = stb->cols->size > MAX_BATCOLS ? MAX_BATCOLS : stb->cols->size; - info->nBatCols = RD(max) + 1; - // random select cnt elements from max - randomFillCols(info->batCols, max, info->nBatCols); + if(stb->partialColNum > 0 && stb->partialColNum < MAX_BATCOLS) { + info->nBatCols = stb->partialColNum; + int j = 0; + for (int i = stb->partialColFrom; i < stb->partialColFrom + stb->partialColNum; i++) { + info->batCols[j++] = i; + } + } else { + info->nBatCols = RD(max) + 1; + // random select cnt elements from max + randomFillCols(info->batCols, max, info->nBatCols); + } + char * colNames = genBatColsNames(info, stb); len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, "%s %s.%s (%s) VALUES ", STR_INSERT_INTO, db->dbName, tableName, colNames); free(colNames); @@ -276,7 +285,7 @@ uint32_t appendRowRuleOld(SSuperTable* stb, char* pstr, uint32_t len, int64_t ti } #define GET_IDX(i) info->batCols[i] -uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts) { +uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts, int64_t* k) { uint32_t size = 0; // first col is ts if (stb->useNow) { @@ -308,7 +317,7 @@ uint32_t genRowMixAll(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t l } } - size += dataGenByField(fd, pstr, len + size, prefix); + size += dataGenByField(fd, pstr, len + size, prefix, k, VAL_NULL); } // end @@ -336,12 +345,12 @@ uint32_t genRowTsCalc(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t l // create columns data -uint32_t createColsData(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts) { +uint32_t createColsData(threadInfo* info, SSuperTable* stb, char* pstr, uint32_t len, int64_t ts, int64_t* k) { uint32_t size = 0; // gen row data if (stb->genRowRule == RULE_MIX_ALL) { - size = genRowMixAll(info, stb, pstr, len, ts); + size = genRowMixAll(info, stb, pstr, len, ts, k); } else if (stb->genRowRule == RULE_MIX_TS_CALC) { size = genRowTsCalc(info, stb, pstr, len, ts); } else { // random @@ -383,7 +392,8 @@ bool takeRowOutToBuf(SMixRatio* mix, uint8_t type, int64_t ts) { // row rule mix , global info put into mix // #define MIN_COMMIT_ROWS 10000 -uint32_t appendRowRuleMix(threadInfo* info, SSuperTable* stb, SMixRatio* mix, char* pstr, uint32_t len, int64_t ts, uint32_t* pGenRows) { +uint32_t appendRowRuleMix(threadInfo* info, SSuperTable* stb, SMixRatio* mix, char* pstr, + uint32_t len, int64_t ts, uint32_t* pGenRows, int64_t *k) { uint32_t size = 0; // remain need generate rows bool forceDis = FORCE_TAKEOUT(MDIS); @@ -398,10 +408,12 @@ uint32_t appendRowRuleMix(threadInfo* info, SSuperTable* stb, SMixRatio* mix, ch } // gen col data - size = createColsData(info, stb, pstr, len, ts); + size = createColsData(info, stb, pstr, len, ts, k); if(size > 0) { + //record counter + *k += 1; *pGenRows += 1; - debugPrint(" row ord ts=%" PRId64 " \n", ts); + debugPrint(" row ord ts=%" PRId64 " k=%"PRId64"\n", ts, *k); } // update @@ -415,7 +427,8 @@ uint32_t appendRowRuleMix(threadInfo* info, SSuperTable* stb, SMixRatio* mix, ch // // fill update rows from mix // -uint32_t fillBatchWithBuf(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t startTime, char* pstr, uint32_t len, uint32_t* pGenRows, uint8_t type, uint32_t maxFill, bool force) { +uint32_t fillBatchWithBuf(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t startTime, char* pstr, + uint32_t len, uint32_t* pGenRows, uint8_t type, uint32_t maxFill, bool force, int64_t *k) { uint32_t size = 0; if (maxFill == 0) return 0; @@ -456,7 +469,7 @@ uint32_t fillBatchWithBuf(threadInfo* info, SSuperTable* stb, SMixRatio* mix, in } // generate row by ts - size += createColsData(info, stb, pstr, len + size, ts); + size += createColsData(info, stb, pstr, len + size, ts, k); *pGenRows += 1; selCnt ++; debugPrint(" row %s ts=%" PRId64 " \n", type == MDIS ? "dis" : "upd", ts); @@ -474,7 +487,8 @@ uint32_t fillBatchWithBuf(threadInfo* info, SSuperTable* stb, SMixRatio* mix, in // // generate insert batch body, return rows in batch // -uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t* pStartTime, char* pstr, uint32_t slen, STotal* pBatT) { +uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t* pStartTime, char* pstr, + uint32_t slen, STotal* pBatT, int32_t *pkCur, int32_t *pkCnt, int64_t *k) { int32_t genRows = 0; int64_t ts = *pStartTime; int64_t startTime = *pStartTime; @@ -505,7 +519,7 @@ uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t infoPrint(" ord found duplicate ts=%" PRId64 " rows=%" PRId64 "\n", ts, pBatT->ordRows); } - len += appendRowRuleMix(info, stb, mix, pstr, len, ts, &ordRows); + len += appendRowRuleMix(info, stb, mix, pstr, len, ts, &ordRows, k); if (ordRows > 0) { genRows += ordRows; pBatT->ordRows += ordRows; @@ -536,7 +550,7 @@ uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t } uint32_t updRows = 0; - len += fillBatchWithBuf(info, stb, mix, startTime, pstr, len, &updRows, MUPD, maxFill, forceUpd); + len += fillBatchWithBuf(info, stb, mix, startTime, pstr, len, &updRows, MUPD, maxFill, forceUpd, k); if (updRows > 0) { genRows += updRows; pBatT->updRows += updRows; @@ -564,7 +578,7 @@ uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t } uint32_t disRows = 0; - len += fillBatchWithBuf(info, stb, mix, startTime, pstr, len, &disRows, MDIS, maxFill, forceDis); + len += fillBatchWithBuf(info, stb, mix, startTime, pstr, len, &disRows, MDIS, maxFill, forceDis, k); if (disRows > 0) { genRows += disRows; pBatT->disRows += disRows; @@ -575,7 +589,9 @@ uint32_t genBatchSql(threadInfo* info, SSuperTable* stb, SMixRatio* mix, int64_t } // if RULE_ // move next ts - ts += timestamp_step; + if (!stb->primary_key || needChangeTs(stb, pkCur, pkCnt)) { + ts += timestamp_step; + } // check over TSDB_MAX_ALLOWED_SQL_LENGTH if (len > (TSDB_MAX_ALLOWED_SQL_LEN - stb->lenOfCols - 320)) { @@ -750,7 +766,6 @@ bool checkCorrect(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tbNam // bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { int64_t lastPrintTime = 0; - // check interface if (stb->iface != TAOSC_IFACE) { return false; @@ -761,6 +776,17 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { return false; } + infoPrint("insert mode is mix. generate_row_rule=%d\n", stb->genRowRule); + + FILE* csvFile = NULL; + char* tagData = NULL; + bool acreate = (stb->genRowRule == RULE_OLD || stb->genRowRule == RULE_MIX_RANDOM) && stb->autoTblCreating; + int w = 0; + if (acreate) { + csvFile = openTagCsv(stb); + tagData = benchCalloc(TAG_BATCH_COUNT, stb->lenOfTags, false); + } + // debug //g_arguments->debug_print = true; @@ -772,13 +798,23 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { // loop insert child tables for (uint64_t tbIdx = info->start_table_from; tbIdx <= info->end_table_to; ++tbIdx) { - char* tbName = stb->childTblArray[tbIdx]->name; + // get child table + SChildTable *childTbl; + if (g_arguments->bind_vgroup) { + childTbl = info->vg->childTblArray[tbIdx]; + } else { + childTbl = stb->childTblArray[tbIdx]; + } + char* tbName = childTbl->name; SMixRatio mixRatio; mixRatioInit(&mixRatio, stb); int64_t batStartTime = stb->startTimestamp; + int32_t pkCur = 0; // primary key repeat ts current count + int32_t pkCnt = 0; // primary key repeat ts count STotal tbTotal; memset(&tbTotal, 0 , sizeof(STotal)); + int64_t k = 0; // position while (mixRatio.insertedRows < mixRatio.insertRows) { // check terminate @@ -786,19 +822,36 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { break; } + if(acreate) { + // generator + if (w == 0) { + if(!generateTagData(stb, tagData, TAG_BATCH_COUNT, csvFile)) { + FAILED_BREAK() + } + } + } + // generate pre sql like "insert into tbname ( part column names) values " - uint32_t len = genInsertPreSql(info, db, stb, tbName, tbIdx, info->buffer); + uint32_t len = genInsertPreSql(info, db, stb, tbName, tagData, tbIdx, info->buffer); + + if(acreate) { + // move next + if (++w >= TAG_BATCH_COUNT) { + // reset for gen again + w = 0; + } + } // batch create sql values STotal batTotal; memset(&batTotal, 0 , sizeof(STotal)); - uint32_t batchRows = genBatchSql(info, stb, &mixRatio, &batStartTime, info->buffer, len, &batTotal); + uint32_t batchRows = genBatchSql(info, stb, &mixRatio, &batStartTime, info->buffer, len, &batTotal, &pkCur, &pkCnt, &k); // execute insert sql int64_t startTs = toolsGetTimestampUs(); //g_arguments->debug_print = false; - if(execInsert(info, batchRows) != 0) { + if(execInsert(info, batchRows, NULL) != 0) { FAILED_BREAK() } //g_arguments->debug_print = true; @@ -848,7 +901,7 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { batTotal.delRows = genBatchDelSql(stb, &mixRatio, batStartTime, info->conn->taos, tbName, info->buffer, len, querySql); if (batTotal.delRows > 0) { // g_arguments->debug_print = false; - if (execInsert(info, batTotal.delRows) != 0) { + if (execInsert(info, batTotal.delRows, NULL) != 0) { FAILED_BREAK() } @@ -959,5 +1012,12 @@ bool insertDataMix(threadInfo* info, SDataBase* db, SSuperTable* stb) { total.ordRows, total.disRows, total.updRows, total.delRows); //g_arguments->debug_print = false; + + // free + if(csvFile) { + fclose(csvFile); + } + tmfree(tagData); + return true; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 8c83bf4a..6481e40e 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -11,22 +11,75 @@ */ #include +#include #include extern char g_configDir[MAX_PATH_LEN]; char funsName [FUNTYPE_CNT] [32] = { "sin(", - "cos(" + "cos(", + "count(", + "saw(", + "square(", + "tri(", }; -uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* random) { +int32_t parseFunArgs(char* value, uint8_t funType, int64_t* min ,int64_t* max, int32_t* step ,int32_t* period ,int32_t* offset) { + char* buf = strdup(value); + char* p[4] = {NULL}; + int32_t i = 0; + // find ")" fun end brance + char* end = strstr(buf,")"); + if(end) { + *end = 0; + } + int32_t argsLen = strlen(buf) + 1; + + // find first + char* token = strtok(buf, ","); + if(token == NULL) { + free(buf); + return 0; + } + p[i++] = token; + + // find others + while((token = strtok(NULL, ",")) && i < 4) { + p[i++] = token; + } + + if(i != 4) { + // must 4 params + free(buf); + return 0; + } + + // parse fun + if(funType == FUNTYPE_COUNT) { + *min = atoi(p[0]); + *max = atoi(p[1]); + *step = atoi(p[2]); + *offset = atoi(p[3]); + } else { + *min = atoi(p[0]); + *max = atoi(p[1]); + *period = atoi(p[2]); + *offset = atoi(p[3]); + } + + free(buf); + return argsLen; +} + +uint8_t parseFuns(char* expr, float* multiple, float* addend, float* base, int32_t* random, + int64_t* min ,int64_t* max, int32_t* step ,int32_t* period ,int32_t* offset) { // check valid - if (funValue == NULL || multiple == NULL || addend == NULL) { + if (expr == NULL || multiple == NULL || addend == NULL || base == NULL) { return FUNTYPE_NONE; } - size_t len = strlen(funValue); + size_t len = strlen(expr); if(len > 100) { return FUNTYPE_NONE; } @@ -34,9 +87,10 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran //parse format 10*sin(x) + 100 * random(5) char value[128]; size_t n = 0; + // remove blank for (size_t i = 0; i < len; i++) { - if (funValue[i] != ' ') { - value[n++] = funValue[i]; + if (expr[i] != ' ') { + value[n++] = expr[i]; } } // set end @@ -44,19 +98,41 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran // multiple char* key1 = strstr(value, "*"); - if(key1 == NULL) return FUNTYPE_NONE; - *key1 = 0; - * multiple = atof(value); - key1 += 1; + if(key1) { + // excpet tri(-20,40,20,5)+20+50*random(12) + bool found = true; + char* p1 = strstr(value+1, "+"); + char* p2 = strstr(value+1, "-"); + if(p1 && key1 > p1 ) + found = false; + if(p2 && key1 > p2 ) + found = false; + + if(found) { + *key1 = 0; + *multiple = atof(value); + key1 += 1; + } else { + key1 = value; + } + } else { + key1 = value; + } // funType uint8_t funType = FUNTYPE_NONE; char* key2 = NULL; - for(int i=0; i < FUNTYPE_CNT; i++) { + for (int i = 0; i < FUNTYPE_CNT - 1; i++) { key2 = strstr(key1, funsName[i]); if(key2) { funType = i + 1; key2 += strlen(funsName[i]); + int32_t argsLen = parseFunArgs(key2, funType, min, max, step, period, offset); + if(len <= 0){ + return FUNTYPE_NONE; + } + key2 += argsLen; + break; } } @@ -65,18 +141,36 @@ uint8_t parseFuns(char* funValue, float* multiple, int32_t* addend, int32_t* ran char* key3 = strstr(key2, "+"); if(key3) { - *addend = atoi(key3 + 1); + *addend = atof(key3 + 1); + key3 += 1; } else { key3 = strstr(key2, "-"); - if(key3) - *addend = atoi(key3 + 1) * -1; + if(key3) { + *addend = atof(key3 + 1) * -1; + key3 += 1; + } } - key3 += 1; + // random - char* key4 = strstr(key3, "*random("); - if(key4) { - *random = atoi(key4 + 8); + if(key3) { + char* key4 = strstr(key3, "*random("); + if(key4) { + *random = atoi(key4 + 8); + key3 += 9; + } + } + + // base + if(key3) { + char* key5 = strstr(key3, "+"); + if(key5){ + *base = atof(key5+1); + } else { + key5 = strstr(key3, "-"); + if(key5) + *base = atof(key5+1) * -1; + } } return funType; @@ -108,8 +202,17 @@ static int getColumnAndTagTypeFromInsertJsonFile( // fun type uint8_t funType = FUNTYPE_NONE; float multiple = 0; - int32_t addend = 0; + float addend = 0; + float base = 0; int32_t random = 0; + int32_t step = 0; + int32_t period = 0; + int32_t offset = 0; + uint8_t gen = GEN_RANDOM; + bool fillNull = true; + char* encode = NULL; + char* compress = NULL; + char* level = NULL; tools_cJSON *column = tools_cJSON_GetArrayItem(columnsObj, k); if (!tools_cJSON_IsObject(column)) { @@ -149,10 +252,41 @@ static int getColumnAndTagTypeFromInsertJsonFile( min = convertDatatypeToDefaultMin(type); } + // gen + tools_cJSON *dataGen = tools_cJSON_GetObjectItem(column, "gen"); + if (tools_cJSON_IsString(dataGen)) { + if (strcasecmp(dataGen->valuestring, "order") == 0) { + gen = GEN_ORDER; + } + } + // fillNull + tools_cJSON *dataNull = tools_cJSON_GetObjectItem(column, "fillNull"); + if (tools_cJSON_IsString(dataNull)) { + if (strcasecmp(dataNull->valuestring, "false") == 0) { + fillNull = false; + } + } + + // encode + tools_cJSON *dataEncode = tools_cJSON_GetObjectItem(column, "encode"); + if (tools_cJSON_IsString(dataEncode)) { + encode = dataEncode->valuestring; + } + // compress + tools_cJSON *dataCompress = tools_cJSON_GetObjectItem(column, "compress"); + if (tools_cJSON_IsString(dataCompress)) { + compress = dataCompress->valuestring; + } + // level + tools_cJSON *dataLevel = tools_cJSON_GetObjectItem(column, "level"); + if (tools_cJSON_IsString(dataLevel)) { + level = dataLevel->valuestring; + } + // fun tools_cJSON *fun = tools_cJSON_GetObjectItem(column, "fun"); if (tools_cJSON_IsString(fun)) { - funType = parseFuns(fun->valuestring, &multiple, &addend, &random); + funType = parseFuns(fun->valuestring, &multiple, &addend, &base, &random, &min, &max, &step, &period, &offset); } tools_cJSON *dataValues = tools_cJSON_GetObjectItem(column, "values"); @@ -191,12 +325,18 @@ static int getColumnAndTagTypeFromInsertJsonFile( col->sma = sma; col->max = max; col->min = min; + col->gen = gen; + col->fillNull = fillNull; col->values = dataValues; // fun col->funType = funType; col->multiple = multiple; col->addend = addend; + col->base = base; col->random = random; + col->step = step; + col->period = period; + col->offset = offset; if (customName) { if (n >= 1) { @@ -209,6 +349,32 @@ static int getColumnAndTagTypeFromInsertJsonFile( } else { snprintf(col->name, TSDB_COL_NAME_LEN, "c%d", index); } + + // encode + if(encode) { + if (strlen(encode) < COMP_NAME_LEN) { + strcpy(col->encode, encode); + } else { + errorPrint("encode name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, encode); + } + } + // compress + if(compress) { + if (strlen(compress) < COMP_NAME_LEN) { + strcpy(col->compress, compress); + } else { + errorPrint("compress name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, compress); + } + } + // level + if(level) { + if (strlen(level) < COMP_NAME_LEN) { + strcpy(col->level, level); + } else { + errorPrint("level name length over (%d) bytes, ignore. name=%s", COMP_NAME_LEN, level); + } + } + index++; } } @@ -575,6 +741,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->insert_interval = g_arguments->insert_interval; superTable->max_sql_len = TSDB_MAX_ALLOWED_SQL_LEN; superTable->partialColNum = 0; + superTable->partialColFrom = 0; superTable->comment = NULL; superTable->delay = -1; superTable->file_factor = -1; @@ -661,9 +828,9 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->iface = REST_IFACE; } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { superTable->iface = STMT_IFACE; - if (g_arguments->reqPerReq > INT16_MAX) { - g_arguments->reqPerReq = INT16_MAX; - } + //if (g_arguments->reqPerReq > INT16_MAX) { + // g_arguments->reqPerReq = INT16_MAX; + //} if (g_arguments->reqPerReq > g_arguments->prepared_rand) { g_arguments->prepared_rand = g_arguments->reqPerReq; } @@ -765,13 +932,23 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { if (tools_cJSON_IsNumber(childTbl_to)) { superTable->childTblTo = childTbl_to->valueint; if (superTable->childTblTo < superTable->childTblFrom) { - errorPrint("child table _to_ is invalid number," + errorPrint("json config invalid. child table _to_ is invalid number," "%"PRId64" < %"PRId64"\n", superTable->childTblTo, superTable->childTblFrom); return -1; } } + // check childtable_from and childtable_to valid + if (superTable->childTblFrom >= superTable->childTblCount) { + errorPrint("json config invalid. childtable_from(%"PRId64") is equal or large than childtable_count(%"PRId64")\n", superTable->childTblFrom, superTable->childTblCount); + return -1; + } + if (superTable->childTblTo > superTable->childTblCount) { + errorPrint("json config invalid. childtable_to(%"PRId64") is large than childtable_count(%"PRId64")\n", superTable->childTblTo, superTable->childTblCount); + return -1; + } + tools_cJSON *continueIfFail = tools_cJSON_GetObjectItem(stbInfo, "continue_if_fail"); // yes, no, if (tools_cJSON_IsString(continueIfFail)) { @@ -788,7 +965,29 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { } } - tools_cJSON *ts = tools_cJSON_GetObjectItem(stbInfo, "start_timestamp"); + // start_fillback_time + superTable->startFillbackTime = 0; + tools_cJSON *ts = tools_cJSON_GetObjectItem(stbInfo, "start_fillback_time"); + if (tools_cJSON_IsString(ts)) { + if(0 == strcasecmp(ts->valuestring, "auto")) { + superTable->autoFillback = true; + superTable->startFillbackTime = 0; + } + else if (toolsParseTime(ts->valuestring, + &(superTable->startFillbackTime), + (int32_t)strlen(ts->valuestring), + database->precision, 0)) { + errorPrint("failed to parse time %s\n", ts->valuestring); + return -1; + } + } else { + if (tools_cJSON_IsNumber(ts)) { + superTable->startFillbackTime = ts->valueint; + } + } + + // start_timestamp + ts = tools_cJSON_GetObjectItem(stbInfo, "start_timestamp"); if (tools_cJSON_IsString(ts)) { if (0 == strcasecmp(ts->valuestring, "now")) { superTable->startTimestamp = @@ -796,6 +995,9 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->useNow = true; // fill time with now conflict with check_sql g_arguments->check_sql = false; + } else if (0 == strncasecmp(ts->valuestring, "now", 3)) { + // like now - 7d expression + superTable->calcNow = ts->valuestring; } else { if (toolsParseTime(ts->valuestring, &(superTable->startTimestamp), @@ -986,6 +1188,12 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->partialColNum = pPartialColNum->valueint; } + tools_cJSON *pPartialColFrom = + tools_cJSON_GetObjectItem(stbInfo, "partial_col_from"); + if (tools_cJSON_IsNumber(pPartialColFrom)) { + superTable->partialColFrom = pPartialColFrom->valueint; + } + if (g_arguments->taosc_version == 3) { tools_cJSON *delay = tools_cJSON_GetObjectItem(stbInfo, "delay"); if (tools_cJSON_IsNumber(delay)) { @@ -1028,6 +1236,39 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { if (getColumnAndTagTypeFromInsertJsonFile(stbInfo, superTable)) { return -1; } + + // primary key + itemObj = tools_cJSON_GetObjectItem(stbInfo, "primary_key"); + if (tools_cJSON_IsNumber(itemObj)) { + superTable->primary_key = itemObj->valueint == 1; + } + // repeat_ts_min + itemObj = tools_cJSON_GetObjectItem(stbInfo, "repeat_ts_min"); + if (tools_cJSON_IsNumber(itemObj)) { + superTable->repeat_ts_min = (int)itemObj->valueint; + } + // repeat_ts_max + itemObj = tools_cJSON_GetObjectItem(stbInfo, "repeat_ts_max"); + if (tools_cJSON_IsNumber(itemObj)) { + superTable->repeat_ts_max = (int)itemObj->valueint; + } + + // sqls + itemObj = tools_cJSON_GetObjectItem(stbInfo, "sqls"); + if (tools_cJSON_IsArray(itemObj)) { + int cnt = tools_cJSON_GetArraySize(itemObj); + if(cnt > 0) { + char ** sqls = (char **)benchCalloc(cnt + 1, sizeof(char *), false); // +1 add end + superTable->sqls = sqls; + for(int j = 0; j < cnt; j++) { + tools_cJSON *sqlObj = tools_cJSON_GetArrayItem(itemObj, j); + if(sqlObj && tools_cJSON_IsString(sqlObj)) { + *sqls = strdup(sqlObj->valuestring); + sqls++; + } + } + } + } } return 0; } @@ -1151,12 +1392,20 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { tools_cJSON *host = tools_cJSON_GetObjectItem(json, "host"); if (host && host->type == tools_cJSON_String && host->valuestring != NULL) { - g_arguments->host = host->valuestring; + if(g_arguments->host && strlen(g_arguments->host) > 0) { + warnPrint("command line already pass host is %s, json config host(%s) had been ignored.\n", g_arguments->host, host->valuestring); + } else { + g_arguments->host = host->valuestring; + } } tools_cJSON *port = tools_cJSON_GetObjectItem(json, "port"); if (port && port->type == tools_cJSON_Number) { - g_arguments->port = (uint16_t)port->valueint; + if(g_arguments->port != DEFAULT_PORT) { + warnPrint("command line already pass port is %d, json config port(%d) had been ignored.\n", g_arguments->port, (uint16_t)port->valueint); + } else { + g_arguments->port = (uint16_t)port->valueint; + } } tools_cJSON *user = tools_cJSON_GetObjectItem(json, "user"); @@ -1196,6 +1445,27 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { } } + g_arguments->csvPath[0] = 0; + tools_cJSON *csv = tools_cJSON_GetObjectItem(json, "csvPath"); + if (csv && (csv->type == tools_cJSON_String) + && (csv->valuestring != NULL)) { + tstrncpy(g_arguments->csvPath, csv->valuestring, MAX_FILE_NAME_LEN); + } + + size_t len = strlen(g_arguments->csvPath); + + if(len == 0) { + // set default with current path + strcpy(g_arguments->csvPath, "./output/"); + mkdir(g_arguments->csvPath, 0775); + } else { + // append end + if (g_arguments->csvPath[len-1] != '/' ) { + strcat(g_arguments->csvPath, "/"); + } + mkdir(g_arguments->csvPath, 0775); + } + code = 0; return code; } @@ -1219,6 +1489,14 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) { } } + g_arguments->pre_load_tb_meta = false; + tools_cJSON *preLoad = tools_cJSON_GetObjectItem(json, "pre_load_tb_meta"); + if (tools_cJSON_IsString(preLoad)) { + if (0 == strcasecmp(preLoad->valuestring, "yes")) { + g_arguments->pre_load_tb_meta = true; + } + } + tools_cJSON *resultfile = tools_cJSON_GetObjectItem(json, "result_file"); if (resultfile && resultfile->type == tools_cJSON_String && resultfile->valuestring != NULL) { @@ -1230,6 +1508,13 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) { g_arguments->nthreads = (uint32_t)threads->valueint; } + tools_cJSON *bindVGroup = tools_cJSON_GetObjectItem(json, "thread_bind_vgroup"); + if (tools_cJSON_IsString(bindVGroup)) { + if (0 == strcasecmp(bindVGroup->valuestring, "yes")) { + g_arguments->bind_vgroup = true; + } + } + tools_cJSON *keepTrying = tools_cJSON_GetObjectItem(json, "keep_trying"); if (keepTrying && keepTrying->type == tools_cJSON_Number) { g_arguments->keep_trying = (int32_t)keepTrying->valueint; @@ -1274,7 +1559,7 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) { g_arguments->reqPerReq = DEFAULT_REQ_PER_REQ; } - if (g_arguments->reqPerReq > 32768) { + if (g_arguments->reqPerReq > INT32_MAX) { infoPrint("warning: num_of_records_per_req item in json config need less than 32768. current = %d. now reset to default.\n", g_arguments->reqPerReq); g_arguments->reqPerReq = DEFAULT_REQ_PER_REQ; } @@ -1993,6 +2278,8 @@ int getInfoFromJsonFile() { g_arguments->test_mode = QUERY_TEST; } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { g_arguments->test_mode = SUBSCRIBE_TEST; + } else if (0 == strcasecmp("csvfile", filetype->valuestring)) { + g_arguments->test_mode = CSVFILE_TEST; } else { errorPrint("%s", "failed to read json, filetype not support\n"); @@ -2004,7 +2291,7 @@ int getInfoFromJsonFile() { // read common item code = getMetaFromCommonJsonFile(root); - if (INSERT_TEST == g_arguments->test_mode) { + if (INSERT_TEST == g_arguments->test_mode || CSVFILE_TEST == g_arguments->test_mode) { code = getMetaFromInsertJsonFile(root); #ifdef TD_VER_COMPATIBLE_3_0_0_0 } else if (QUERY_TEST == g_arguments->test_mode) { diff --git a/src/benchMain.c b/src/benchMain.c index 46b461c6..04b21191 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -11,6 +11,7 @@ */ #include +#include #include SArguments* g_arguments; @@ -86,7 +87,6 @@ int main(int argc, char* argv[]) { if (dsn != NULL) { g_arguments->dsn = dsn; g_arguments->websocket = true; - g_arguments->nthreads_auto = false; } else { g_arguments->dsn = false; } @@ -99,11 +99,17 @@ int main(int argc, char* argv[]) { modifyArgument(); } - g_arguments->fpOfInsertResult = fopen(g_arguments->output_file, "a"); - if (NULL == g_arguments->fpOfInsertResult) { - errorPrint("failed to open %s for save result\n", - g_arguments->output_file); + if(g_arguments->output_file[0] == 0) { + infoPrint("%s","result_file is empty, ignore output."); + g_arguments->fpOfInsertResult = NULL; + } else { + g_arguments->fpOfInsertResult = fopen(g_arguments->output_file, "a"); + if (NULL == g_arguments->fpOfInsertResult) { + errorPrint("failed to open %s for save result\n", + g_arguments->output_file); + } } + infoPrint("client version: %s\n", taos_get_client_info()); if (g_arguments->test_mode == INSERT_TEST) { @@ -111,6 +117,11 @@ int main(int argc, char* argv[]) { errorPrint("%s", "insert test process failed\n"); ret = -1; } + } else if (g_arguments->test_mode == CSVFILE_TEST) { + if (csvTestProcess()) { + errorPrint("%s", "query test process failed\n"); + ret = -1; + } } else if (g_arguments->test_mode == QUERY_TEST) { if (queryTestProcess(g_arguments)) { errorPrint("%s", "query test process failed\n"); diff --git a/src/benchSys.c b/src/benchSys.c index 456bf3af..d848155a 100644 --- a/src/benchSys.c +++ b/src/benchSys.c @@ -77,7 +77,7 @@ void benchPrintHelp() { printf("%s%s%s%s\r\n", indent, "-r,", indent, BENCH_BATCH); printf("%s%s%s%s\r\n", indent, "-R,", indent, BENCH_RANGE); printf("%s%s%s%s\r\n", indent, "-S,", indent, BENCH_STEP); - printf("%s%s%s%s\r\n", indent, "-s,", indent, BENCH_SUPPLEMENT); + printf("%s%s%s%s\r\n", indent, "-s,", indent, BENCH_START_TIMESTAMP); printf("%s%s%s%s\r\n", indent, "-t,", indent, BENCH_TABLE); printf("%s%s%s%s\r\n", indent, "-T,", indent, BENCH_THREAD); printf("%s%s%s%s\r\n", indent, "-u,", indent, BENCH_USER); @@ -167,7 +167,7 @@ int32_t benchParseArgsNoArgp(int argc, char* argv[]) { return 0; } #else -const char * argp_program_version = version; +//const char * argp_program_version = version; const char * argp_program_bug_address = CUS_EMAIL; static struct argp_option bench_options[] = { @@ -204,7 +204,7 @@ static struct argp_option bench_options[] = { {"answer-yes", 'y', 0, 0, BENCH_YES}, {"disorder-range", 'R', "NUMBER", 0, BENCH_RANGE}, {"disorder", 'O', "NUMBER", 0, BENCH_DISORDER}, - {"replia", 'a', "NUMBER", 0, BENCH_REPLICA}, + {"replica", 'a', "NUMBER", 0, BENCH_REPLICA}, {"debug", 'g', 0, 0, BENCH_DEBUG}, {"performance", 'G', 0, 0, BENCH_PERFORMANCE}, {"prepared_rand", 'F', "NUMBER", 0, BENCH_PREPARE}, @@ -217,7 +217,7 @@ static struct argp_option bench_options[] = { #ifdef TD_VER_COMPATIBLE_3_0_0_0 {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, #endif -// {"version", 'V', 0, 0, BENCH_VERSION}, + {"version", 'V', 0, 0, BENCH_VERSION}, {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} }; @@ -254,13 +254,11 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { case 'f': g_arguments->demo_mode = false; g_arguments->metaFile = arg; - g_arguments->nthreads_auto = false; break; case 'h': g_arguments->host = arg; g_arguments->host_auto = false; - g_arguments->nthreads_auto = false; break; case 'P': @@ -290,7 +288,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { stbInfo->iface = STMT_IFACE; } else if (0 == strcasecmp(arg, "rest")) { stbInfo->iface = REST_IFACE; - g_arguments->nthreads_auto = false; if (false == g_arguments->port_inputted) { g_arguments->port = DEFAULT_REST_PORT; } @@ -311,19 +308,15 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { || (0 == strcasecmp(arg, "sml-rest-line"))) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_LINE_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-telnet")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_TELNET_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-json")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = TSDB_SML_JSON_PROTOCOL; - g_arguments->nthreads_auto = false; } else if (0 == strcasecmp(arg, "sml-rest-taosjson")) { stbInfo->iface = SML_REST_IFACE; stbInfo->lineProtocol = SML_JSON_TAOS_FORMAT; - g_arguments->nthreads_auto = false; } else { errorPrint( "Invalid -I: %s, will auto set to default (taosc)\n", @@ -361,8 +354,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { "Invalid -T: %s, will auto set to default(8)\n", arg); g_arguments->nthreads = DEFAULT_NTHREADS; - } else { - g_arguments->nthreads_auto = false; } break; @@ -448,7 +439,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { case 'U': g_arguments->supplementInsert = true; - g_arguments->nthreads_auto = false; break; case 't': @@ -580,7 +570,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { case 'M': g_arguments->mistMode = true; - g_arguments->prepared_rand = 57; break; case 'x': @@ -653,7 +642,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { #ifdef WEBSOCKET case 'W': - g_arguments->nthreads_auto = false; g_arguments->dsn = arg; break; @@ -670,7 +658,6 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "v"); } - g_arguments->nthreads_auto = false; g_arguments->inputted_vgroups = atoi(arg); break; #endif diff --git a/src/benchTmq.c b/src/benchTmq.c index c92c6f9e..a8cb9d4d 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -199,12 +199,12 @@ static void* tmqConsume(void* arg) { SConsumerInfo* pConsumerInfo = &g_tmqInfo.consumerInfo; // "sequential" or "parallel" - if (0 != strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { + if (pConsumerInfo->createMode && 0 != strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { char* tPtr = pConsumerInfo->groupId; // "share" or "independent" char groupId[16] = {0}; - if (0 != strncasecmp(pConsumerInfo->groupMode, "share", 5)) { + if (pConsumerInfo->groupMode && 0 != strncasecmp(pConsumerInfo->groupMode, "share", 5)) { if ((NULL == pConsumerInfo->groupId) || (0 == strlen(pConsumerInfo->groupId))) { // rand string @@ -312,7 +312,7 @@ int subscribeTestProcess() { } // "share" or "independent" - if (0 == strncasecmp(pConsumerInfo->groupMode, "share", 5)) { + if (pConsumerInfo->groupMode && 0 == strncasecmp(pConsumerInfo->groupMode, "share", 5)) { char groupId[16] = {0}; if ((NULL == pConsumerInfo->groupId) || (0 == strlen(pConsumerInfo->groupId))) { // rand string @@ -334,7 +334,7 @@ int subscribeTestProcess() { pThreadInfo->totalRows = 0; pThreadInfo->id = i; - if (strlen(pConsumerInfo->rowsFile)) { + if ( pConsumerInfo->rowsFile && strlen(pConsumerInfo->rowsFile)) { memset(tmpBuff, 0, sizeof(tmpBuff)); snprintf(tmpBuff, 64, "%s_%d", pConsumerInfo->rowsFile, i); pThreadInfo->fpOfRowsFile = fopen(tmpBuff, "a"); @@ -346,7 +346,7 @@ int subscribeTestProcess() { } // "sequential" or "parallel" - if (0 == strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { + if (pConsumerInfo->createMode && 0 == strncasecmp(pConsumerInfo->createMode, "sequential", 10)) { int retVal = buildConsumerAndSubscribe(pThreadInfo, pConsumerInfo->groupId); if (0 != retVal) { infoPrint("%s\n", "buildConsumerAndSubscribe() fail!"); diff --git a/src/benchUtil.c b/src/benchUtil.c index 4b54854d..50524289 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -41,7 +41,6 @@ FORCE_INLINE void tmfclose(FILE *fp) { FORCE_INLINE void tmfree(void *buf) { if (NULL != buf) { free(buf); - buf = NULL; } } @@ -267,7 +266,10 @@ int regexMatch(const char *s, const char *reg, int cflags) { return 0; } -SBenchConn* initBenchConn() { + + + +SBenchConn* initBenchConnImpl() { SBenchConn* conn = benchCalloc(1, sizeof(SBenchConn), true); #ifdef WEBSOCKET if (g_arguments->websocket) { @@ -309,6 +311,25 @@ SBenchConn* initBenchConn() { return conn; } +SBenchConn* initBenchConn() { + + SBenchConn* conn = NULL; + int32_t keep_trying = 0; + while(1) { + conn = initBenchConnImpl(); + if(conn || ++keep_trying > g_arguments->keep_trying || g_arguments->terminate) { + break; + } + + infoPrint("sleep %dms and try to connect... %d \n", g_arguments->trying_interval, keep_trying); + if(g_arguments->trying_interval > 0) { + toolsMsleep(g_arguments->trying_interval); + } + } + + return conn; +} + void closeBenchConn(SBenchConn* conn) { if(conn == NULL) return ; @@ -774,38 +795,47 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, return code; } +// fetch result fo file or nothing void fetchResult(TAOS_RES *res, threadInfo *pThreadInfo) { - TAOS_ROW row = NULL; - int num_rows = 0; - int num_fields = taos_field_count(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - - char *databuf = (char *)benchCalloc(1, FETCH_BUFFER_SIZE, true); - - int64_t totalLen = 0; + TAOS_ROW row = NULL; + int num_fields = 0; + int64_t totalLen = 0; + TAOS_FIELD *fields = 0; + char *databuf = NULL; + bool toFile = strlen(pThreadInfo->filePath) > 0; + + if(toFile) { + num_fields = taos_field_count(res); + fields = taos_fetch_fields(res); + databuf = (char *)benchCalloc(1, FETCH_BUFFER_SIZE, true); + } // fetch the records row by row while ((row = taos_fetch_row(res))) { - if (totalLen >= (FETCH_BUFFER_SIZE - HEAD_BUFF_LEN * 2)) { - if (strlen(pThreadInfo->filePath) > 0) { + if (toFile) { + if (totalLen >= (FETCH_BUFFER_SIZE - HEAD_BUFF_LEN * 2)) { + // buff is full appendResultBufToFile(databuf, pThreadInfo->filePath); + totalLen = 0; + memset(databuf, 0, FETCH_BUFFER_SIZE); } - totalLen = 0; - memset(databuf, 0, FETCH_BUFFER_SIZE); + + // format row + char temp[HEAD_BUFF_LEN] = {0}; + int len = taos_print_row(temp, row, fields, num_fields); + len += snprintf(temp + len, HEAD_BUFF_LEN - len, "\n"); + debugPrint("query result:%s\n", temp); + memcpy(databuf + totalLen, temp, len); + totalLen += len; } - num_rows++; - char temp[HEAD_BUFF_LEN] = {0}; - int len = taos_print_row(temp, row, fields, num_fields); - len += snprintf(temp + len, HEAD_BUFF_LEN - len, "\n"); - debugPrint("query result:%s\n", temp); - memcpy(databuf + totalLen, temp, len); - totalLen += len; + //if not toFile , only loop call taos_fetch_row } - if (strlen(pThreadInfo->filePath) > 0) { + // end + if (toFile) { appendResultBufToFile(databuf, pThreadInfo->filePath); + free(databuf); } - free(databuf); } char *convertDatatypeToString(int type) { @@ -974,8 +1004,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strcasecmp(type, "varchar")) { return TSDB_DATA_TYPE_BINARY; - } else if (0 == strcasecmp(type, "geometry")) { - return TSDB_DATA_TYPE_GEOMETRY; + } else if (0 == strcasecmp(type, "varbinary")) { + return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1013,8 +1043,8 @@ int convertStringToDatatype(char *type, int length) { return TSDB_DATA_TYPE_JSON; } else if (0 == strncasecmp(type, "varchar", length)) { return TSDB_DATA_TYPE_BINARY; - } else if (0 == strcasecmp(type, "geometry")) { - return TSDB_DATA_TYPE_GEOMETRY; + } else if (0 == strncasecmp(type, "varbinary", length)) { + return TSDB_DATA_TYPE_VARBINARY; } else { errorPrint("unknown data type: %s\n", type); exit(EXIT_FAILURE); @@ -1229,8 +1259,22 @@ FORCE_INLINE void printErrCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res) { taos_free_result(res); } -FORCE_INLINE void printWarnCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res) { - warnPrint("failed to run command %s, code: 0x%08x, reason: %s\n", - cmd, code, taos_errstr(res)); - taos_free_result(res); +int32_t benchGetTotalMemory(int64_t *totalKB) { +#ifdef WINDOWS + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return -1; + } + + *totalKB = memsStat.ullTotalPhys / 1024; + return 0; +#elif defined(_TD_DARWIN_64) + *totalKB = 0; + return 0; +#else + int64_t tsPageSizeKB = sysconf(_SC_PAGESIZE) / 1024; + *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + return 0; +#endif } diff --git a/src/taosdump.c b/src/taosdump.c index e0fc0ecb..334f76d7 100644 --- a/src/taosdump.c +++ b/src/taosdump.c @@ -82,6 +82,9 @@ #define TAOSDUMP_STATUS "unknown" #endif +#ifndef TD_PRODUCT_NAME +#define TD_PRODUCT_NAME "TDengine" +#endif // use 256 as normal buffer length #define BUFFER_LEN 256 @@ -330,8 +333,8 @@ typedef struct { int32_t threadIndex; SDbInfo *dbInfo; char stbName[TSDB_TABLE_NAME_LEN]; - TableDes *stbTableDes; - char *tbNameArr; + TableDes *stbDes; + char **tbNameArr; int precision; void *taos; uint64_t count; @@ -657,6 +660,9 @@ static uint64_t getUniqueIDFromEpoch() { return id; } +// libtaos.so +extern char buildinfo[]; + static void printVersion(FILE *file) { char taostools_longver[] = TAOSDUMP_TAG; char taosdump_status[] = TAOSDUMP_STATUS; @@ -666,12 +672,13 @@ static void printVersion(FILE *file) { char *taostools_ver = strsep(&running, "-"); char taosdump_commit[] = TAOSDUMP_COMMIT_SHA1; - if (strlen(taosdump_status) == 0) { - fprintf(file, "version %s, commit: %s\n", - taostools_ver, taosdump_commit); - } else { - fprintf(file, "version %s, commit: %s, status:%s\n", - taostools_ver, taosdump_commit, taosdump_status); + + fprintf(file,"%s\ntaosdump version: %s\ngit: %s\n", TD_PRODUCT_NAME, taostools_ver, taosdump_commit); +#ifdef LINUX + printf("build: %s\n ", buildinfo); +#endif + if (strlen(taosdump_status) > 0) { + fprintf(file, "status:%s\n", taosdump_status); } free(dupSeq); @@ -1148,7 +1155,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static error_t parse_opt(int key, char *arg, struct argp_state *state); static struct argp argp = {options, parse_opt, args_doc, doc}; -static void freeTbDes(TableDes *tableDes) { +static void freeTbDes(TableDes *tableDes, bool self) { if (NULL == tableDes) return; for (int i = 0; i < (tableDes->columns+tableDes->tags); i++) { @@ -1157,7 +1164,9 @@ static void freeTbDes(TableDes *tableDes) { } } - free(tableDes); + if(self) { + free(tableDes); + } } #ifdef WEBSOCKET @@ -2061,20 +2070,42 @@ static int dumpCreateMTableClause( } #ifdef WEBSOCKET -static int64_t getNtbCountOfStbWS(char *command) { +static int64_t getNtbCountOfStbWS(char* dbName, const char* stbName) { WS_TAOS *ws_taos; if (NULL == (ws_taos = wsConnect())) { - free(command); return -1; } int64_t count = 0; + char *command = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN); + if (NULL == command) { + errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); + return -1; + } + if (3 == g_majorVersionOfClient) { + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + g_args.db_escape_char + ? "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " + "FROM `%s`.%s%s%s)" + : "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " + "FROM %s.%s%s%s)", + dbName, g_escapeChar, stbName, g_escapeChar); + } else { + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + g_args.db_escape_char + ? "SELECT COUNT(TBNAME) FROM `%s`.%s%s%s" + : "SELECT COUNT(TBNAME) FROM %s.%s%s%s", + dbName, g_escapeChar, stbName, g_escapeChar); + } + debugPrint("get stable child count %s", command); + WS_RES *ws_res = ws_query_timeout(ws_taos, command, g_args.ws_timeout); int32_t ws_code = ws_errno(ws_res); if (ws_code) { return cleanIfQueryFailedWS(__func__, __LINE__, command, ws_res); } + tfree(command); while (true) { int rows = 0; @@ -2112,24 +2143,45 @@ static int64_t getNtbCountOfStbWS(char *command) { ws_free_result(ws_res); ws_close(ws_taos); - free(command); return count; } #endif // WEBSOCKET -static int64_t getNtbCountOfStbNative( - const char *dbName, const char *stbName, char *command) { +static int64_t getTbCountOfStbNative(const char *dbName, const char *stbName) { TAOS *taos; if (NULL == (taos = taosConnect(dbName))) { - free(command); return -1; } + char *command = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN); + if (NULL == command) { + errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); + return -1; + } + + if (3 == g_majorVersionOfClient) { + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + g_args.db_escape_char + ? "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " + "FROM `%s`.%s%s%s)" + : "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " + "FROM %s.%s%s%s)", + dbName, g_escapeChar, stbName, g_escapeChar); + } else { + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + g_args.db_escape_char + ? "SELECT COUNT(TBNAME) FROM `%s`.%s%s%s" + : "SELECT COUNT(TBNAME) FROM %s.%s%s%s", + dbName, g_escapeChar, stbName, g_escapeChar); + } + debugPrint("get stable child count %s", command); + int64_t count = 0; TAOS_RES *res = taos_query(taos, command); int32_t code = taos_errno(res); if (code != 0) { cleanIfQueryFailed(__func__, __LINE__, command, res); + free(command); taos_close(taos); return -1; } @@ -2140,8 +2192,7 @@ static int64_t getNtbCountOfStbNative( count = *(int64_t*)row[TSDB_SHOW_TABLES_NAME_INDEX]; } - debugPrint("%s() LN%d, COUNT(TBNAME): %"PRId64"\n", - __func__, __LINE__, count); + infoPrint("Get super table (%s) child tables (%"PRId64") ok\n", stbName, count); taos_free_result(res); taos_close(taos); @@ -3390,7 +3441,7 @@ static avro_value_iface_t* prepareAvroWface( char *jsonSchema, avro_schema_t *schema, RecordSchema **recordSchema, - avro_file_writer_t *db) { + avro_file_writer_t *writer) { ASSERT(avroFilename); if (avro_schema_from_json_length(jsonSchema, strlen(jsonSchema), schema)) { errorPrint("%s() LN%d, Unable to parse:\n%s \nto schema\n" @@ -3420,7 +3471,7 @@ static avro_value_iface_t* prepareAvroWface( } int rval = avro_file_writer_create_with_codec - (avroFilename, *schema, db, g_avro_codec[g_args.avro_codec], 70*1024); + (avroFilename, *schema, writer, g_avro_codec[g_args.avro_codec], 70*1024); if (rval) { errorPrint("There was an error creating %s. reason: %s\n", avroFilename, avro_strerror()); @@ -4899,7 +4950,6 @@ static int processValueToAvro( avro_value_set_double(&branch, GET_DOUBLE_VAL(value)); } break; - case TSDB_DATA_TYPE_BINARY: if (NULL == value) { avro_value_set_branch(&avro_value, 0, &branch); @@ -5106,7 +5156,7 @@ static int64_t writeResultToAvroWS( currentPercent = ((offset) * 100 / queryCount); if (currentPercent > percentComplete) { - infoPrint("%d%% of %s\n", currentPercent, tbName); + //infoPrint("%d%% of %s\n", currentPercent, tbName); percentComplete = currentPercent; } } while (offset < queryCount); @@ -6123,7 +6173,7 @@ static int64_t dumpInAvroTbTagsImpl( avro_value_decref(&value); avro_value_iface_decref(value_class); - freeTbDes(tableDes); + freeTbDes(tableDes, true); free(sqlstr); if (failed) @@ -7370,7 +7420,7 @@ static int64_t dumpInAvroDataImpl( avro_value_iface_decref(value_class); tfree(bindArray); tfree(stmtBuffer); - freeTbDes(tableDes); + freeTbDes(tableDes, true); #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { ws_stmt_close(ws_stmt); @@ -7637,7 +7687,7 @@ static void* dumpInAvroWorkThreadFp(void *arg) { case AVRO_TBTAGS: atomic_add_fetch_64(&g_totalDumpInStbFailed, rows); errorPrint("[%d] %"PRId64"" - "table(s) belong stb from the file(%s) failed to dumped in!\n", + " table(s) belong stb from the file(%s) failed to dumped in!\n", pThreadInfo->threadIndex, rows, fileList[pThreadInfo->from + i]); break; @@ -7645,7 +7695,7 @@ static void* dumpInAvroWorkThreadFp(void *arg) { case AVRO_NTB: atomic_add_fetch_64(&g_totalDumpInNtbFailed, rows); errorPrint("[%d] %"PRId64" " - "normal tables from (%s) failed to dumped in!\n", + " normal tables from (%s) failed to dumped in!\n", pThreadInfo->threadIndex, rows, fileList[pThreadInfo->from + i]); break; @@ -8556,7 +8606,7 @@ static int64_t dumpNormalTable( const SDbInfo *dbInfo, const bool belongStb, const char *stable, - const TableDes *stbTableDes, + const TableDes *stbDes, const char *tbName, const int precision, char *dumpFilename, @@ -8656,6 +8706,9 @@ static int64_t dumpNormalTable( } } + // + // dump out data + // int64_t totalRows = 0; if (!g_args.schemaonly) { if (g_args.avro) { @@ -8672,14 +8725,14 @@ static int64_t dumpNormalTable( numColsAndTags = getTableDesFromStbWS( (WS_TAOS*)taos, dbInfo->name, - stbTableDes, + stbDes, tbName, &tableDes); } else { #endif numColsAndTags = getTableDesFromStbNative( taos, dbInfo->name, - stbTableDes, tbName, &tableDes); + stbDes, tbName, &tableDes); #ifdef WEBSOCKET } #endif @@ -8687,7 +8740,7 @@ static int64_t dumpNormalTable( errorPrint("%s() LN%d columns/tags count is %d\n", __func__, __LINE__, numColsAndTags); if (tableDes) { - freeTbDes(tableDes); + freeTbDes(tableDes, true); return -1; } } @@ -8709,7 +8762,7 @@ static int64_t dumpNormalTable( } if (tableDes) { - freeTbDes(tableDes); + freeTbDes(tableDes, true); } return totalRows; } @@ -8789,7 +8842,7 @@ static int createMTableAvroHeadImp( const char *stable, const TableDes *stbTableDes, const char *tbName, - avro_file_writer_t db, + avro_file_writer_t writer, avro_value_iface_t *wface) { if (0 == strlen(tbName)) { errorPrint("%s() LN%d, pass wrong tbname\n", __func__, __LINE__); @@ -8864,7 +8917,7 @@ static int createMTableAvroHeadImp( errorPrint("%s() LN%d, columns count is %d\n", __func__, __LINE__, colCount); if (subTableDes) { - freeTbDes(subTableDes); + freeTbDes(subTableDes, true); } return -1; } @@ -9177,13 +9230,13 @@ static int createMTableAvroHeadImp( } } - if (0 != avro_file_writer_append_value(db, &record)) { + if (0 != avro_file_writer_append_value(writer, &record)) { errorPrint("%s() LN%d, Unable to write record to file. Message: %s\n", __func__, __LINE__, avro_strerror()); } avro_value_decref(&record); - freeTbDes(subTableDes); + freeTbDes(subTableDes, true); return 0; } @@ -9222,7 +9275,7 @@ static int createMTableAvroHeadSpecified( __func__, __LINE__); tfree(jsonTagsSchema); - freeTbDes(stbTableDes); + freeTbDes(stbTableDes, true); return -1; } @@ -9249,7 +9302,7 @@ static int createMTableAvroHeadSpecified( avro_schema_decref(schema); tfree(jsonTagsSchema); - freeTbDes(stbTableDes); + freeTbDes(stbTableDes, true); return 0; } @@ -9258,7 +9311,7 @@ static int createMTableAvroHeadSpecified( static int64_t fillTbNameArrWS( WS_TAOS *ws_taos, char *command, - char *tbNameArr, + char **tbNameArr, const char *stable, const int64_t preCount) { WS_RES *ws_res = ws_query_timeout(ws_taos, command, g_args.ws_timeout); @@ -9301,12 +9354,13 @@ static int64_t fillTbNameArrWS( debugPrint("%s() LN%d, ws_get_value_in_blocK() return %s. len: %d\n", __func__, __LINE__, (char *)value0, len); } - strncpy(tbNameArr + ntbCount * TSDB_TABLE_NAME_LEN, - (char*)value0, min(TSDB_TABLE_NAME_LEN, len)); + + tbNameArr[ntbCount] = calloc(len+1, 1); + strncpy(tbNameArr[ntbCount], (char*)value0, len); debugPrint("%s() LN%d, sub table name: %s %"PRId64" of stable: %s\n", __func__, __LINE__, - tbNameArr + ntbCount * TSDB_TABLE_NAME_LEN, + tbNameArr[ntbCount], ntbCount, stable); ++ntbCount; @@ -9337,7 +9391,7 @@ static int64_t fillTbNameArrWS( static int64_t fillTbNameArrNative( TAOS *taos, char *command, - char *tbNameArr, + char **tbNameArr, const char *stable, const int64_t preCount) { TAOS_RES *res = taos_query(taos, command); @@ -9347,30 +9401,32 @@ static int64_t fillTbNameArrNative( } TAOS_ROW row = NULL; - int64_t ntbCount = 0; + int64_t n = 0; int currentPercent = 0; int percentComplete = 0; while ((row = taos_fetch_row(res)) != NULL) { int32_t *lengths = taos_fetch_lengths(res); - if (lengths[TSDB_SHOW_TABLES_NAME_INDEX] <= 0) { + // calc name len + int32_t len = lengths[TSDB_SHOW_TABLES_NAME_INDEX]; + if (len <= 0) { errorPrint("%s() LN%d, fetch_row() get %d length!\n", - __func__, __LINE__, lengths[TSDB_SHOW_TABLES_NAME_INDEX]); + __func__, __LINE__, len); continue; } + // malloc and copy + tbNameArr[n] = calloc(len + 1, 1); // add string end + strncpy(tbNameArr[n], (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], len); - strncpy(tbNameArr + ntbCount * TSDB_TABLE_NAME_LEN, - (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - min(TSDB_TABLE_NAME_LEN-1, - lengths[TSDB_SHOW_TABLES_NAME_INDEX])); - - debugPrint("sub table name: %s. %"PRId64" of stable: %s\n", - tbNameArr + ntbCount * TSDB_TABLE_NAME_LEN, - ntbCount, stable); - ++ntbCount; + debugPrint("child table name: %s. %"PRId64" of stable: %s\n", + tbNameArr[n], n, stable); + // tb count add and check + if(++n == preCount) { + break; + } - currentPercent = (ntbCount * 100 / preCount); + currentPercent = (n * 100 / preCount); if (currentPercent > percentComplete) { infoPrint("connection %p fetched %d%% of %s' tbname\n", @@ -9379,79 +9435,29 @@ static int64_t fillTbNameArrNative( } } - if ((preCount > 0) && (percentComplete < 100)) { - errorPrint("%d%% - total %"PRId64" sub-table's names of stable: %s fetched\n", - percentComplete, ntbCount, stable); + if (preCount == n) { + okPrint("total %"PRId64" sub-table's name of stable: %s fetched\n", n, stable); } else { - okPrint("total %"PRId64" sub-table's name of stable: %s fetched\n", - ntbCount, stable); + errorPrint("%d%% - total %"PRId64" sub-table's names of stable: %s fetched\n", + percentComplete, n, stable); } taos_free_result(res); - free(command); - return ntbCount; + return n; } static int64_t fillTbNameArr( void *taos, char **tbNameArr, const SDbInfo *dbInfo, - const char *stable) { - char *command = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN); - if (NULL == command) { - errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); - return -1; - } - - if (3 == g_majorVersionOfClient) { - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - g_args.db_escape_char - ? "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " - "FROM `%s`.%s%s%s)" - : "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " - "FROM %s.%s%s%s)", - dbInfo->name, g_escapeChar, stable, g_escapeChar); - } else { - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - g_args.db_escape_char - ? "SELECT COUNT(TBNAME) FROM `%s`.%s%s%s" - : "SELECT COUNT(TBNAME) FROM %s.%s%s%s", - dbInfo->name, g_escapeChar, stable, g_escapeChar); - } - - infoPrint("Getting tables' number of super table (%s) ...\n", stable); - - int64_t preCount = 0; -#ifdef WEBSOCKET - if (g_args.cloud || g_args.restful) { - preCount = getNtbCountOfStbWS(command); - } else { -#endif - preCount = getNtbCountOfStbNative(dbInfo->name, stable, command); -#ifdef WEBSOCKET - } -#endif - - if (0 == preCount) { - infoPrint("%s() Tables number is ZERO!\n", __func__); - } else if (0 > preCount) { - errorPrint("Failed to get count of normal table of %s!\n", stable); - } - infoPrint("The number of tables of %s is %"PRId64"!\n", stable, preCount); - + const char *stable, + int64_t preCount) { + // char *command2 = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN); if (NULL == command2) { errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); return -1; } - *tbNameArr = calloc(preCount, TSDB_TABLE_NAME_LEN); - if (NULL == *tbNameArr) { - errorPrint("%s() LN%d, memory allocation failed!\n", - __func__, __LINE__); - free(command2); - return -1; - } - if (3 == g_majorVersionOfClient) { snprintf(command2, TSDB_MAX_ALLOWED_SQL_LEN, g_args.db_escape_char @@ -9474,139 +9480,685 @@ static int64_t fillTbNameArr( #ifdef WEBSOCKET if (g_args.cloud || g_args.restful) { ntbCount = fillTbNameArrWS( - taos, command2, *tbNameArr, stable, preCount); + taos, command2, tbNameArr, stable, preCount); } else { #endif ntbCount = fillTbNameArrNative( - taos, command2, *tbNameArr, stable, preCount); + taos, command2, tbNameArr, stable, preCount); #ifdef WEBSOCKET } #endif infoPrint("The number of tables of %s be filled is %"PRId64"!\n", stable, ntbCount); + free(command2); return ntbCount; } -static int createMTableAvroHead( - void *taos, - const SDbInfo *dbInfo, - const char *stable, - TableDes **pTableDes, - char **tbNameArr) { - if (0 == strlen(stable)) { - errorPrint("%s() LN%d, pass wrong tbname\n", __func__, __LINE__); - return -1; - } - TableDes *tableDes = *pTableDes; +// old createMTableAvroHeadImp +static int writeTagsToAvro( + const char *dbName, + const TableDes *stbDes, + const TableDes *tbDes, + avro_file_writer_t writer, + avro_value_iface_t *wface) { + // avro + avro_value_t record; + avro_generic_value_new(wface, &record); + avro_value_t value, branch; - char dumpFilename[MAX_PATH_LEN] = {0}; + if (!g_args.loose_mode) { + if (0 != avro_value_get_by_name( + &record, "stbname", &value, NULL)) { + errorPrint("%s() LN%d, avro_value_get_by_name(..%s..) failed", + __func__, __LINE__, "stbname"); + return -1; + } - if (0 != generateFilename(AVRO_TBTAGS, dumpFilename, - dbInfo, stable, 0)) { - return -1; + avro_value_set_branch(&value, 1, &branch); + char* outSName = (char*)stbDes->name; + char stableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(stableName, (char*)stbDes->name)) { + outSName = stableName; + } + avro_value_set_string(&branch, outSName); } - debugPrint("%s() LN%d dumpFilename: %s\n", - __func__, __LINE__, dumpFilename); - char *jsonTagsSchema = NULL; - if (0 != convertTbTagsDesToJsonWrap( - dbInfo->name, stable, tableDes, &jsonTagsSchema)) { - errorPrint("%s() LN%d, convertTbTagsDesToJsonWrap failed\n", - __func__, - __LINE__); - tfree(jsonTagsSchema); + if (0 != avro_value_get_by_name( + &record, "tbname", &value, NULL)) { + errorPrint("%s() LN%d, avro_value_get_by_name(..%s..) failed", + __func__, __LINE__, "tbname"); return -1; } - debugPrint("tagsJson:\n%s\n", jsonTagsSchema); - - avro_schema_t schema; - RecordSchema *recordSchema; - avro_file_writer_t db; - - avro_value_iface_t *wface = prepareAvroWface( - dumpFilename, - jsonTagsSchema, &schema, &recordSchema, &db); - - infoPrint("connection: %p is dumping out schema of " - "sub-table(s) of %s \n", - taos, stable); - - int64_t ntbCount = fillTbNameArr( - taos, tbNameArr, dbInfo, stable); + avro_value_set_branch(&value, 1, &branch); - if (ntbCount < 0) { - if (*tbNameArr) { - free(*tbNameArr); - } - avro_value_iface_decref(wface); - freeRecordSchema(recordSchema); - avro_file_writer_close(db); - avro_schema_decref(schema); - tfree(jsonTagsSchema); - return -1; + char* outName = (char*)tbDes->name; + char tableName[TSDB_TABLE_NAME_LEN + 1]; + if(g_args.dotReplace && replaceCopy(tableName, (char*)tbDes->name)) { + outName = tableName; } + avro_value_set_string(&branch, outName); - int currentPercent = 0; - int percentComplete = 0; - - int64_t tb = 0; - infoPrint("connection %p is dumping out schema:%d%% of %s\n", - taos, currentPercent, stable); - for (; tb < ntbCount; tb++) { - createMTableAvroHeadImp( - taos, - dbInfo->name, - stable, - tableDes, - *tbNameArr + tb*TSDB_TABLE_NAME_LEN, - db, wface); + for (int tag = 0; tag < tbDes->tags; tag++) { + debugPrint("%s() LN%d, sub table %s no. %d tags is %s, " + "type is %d, value is %s\n", + __func__, __LINE__, tbDes->name, tag, + tbDes->cols[tbDes->columns + tag].field, + tbDes->cols[tbDes->columns + tag].type, + tbDes->cols[tbDes->columns + tag].value); - currentPercent = ((tb+1) * 100 / ntbCount); + char tmpBuf[MIDDLE_BUFF_LEN] = {0}; + snprintf(tmpBuf, MIDDLE_BUFF_LEN, "tag%d", tag); - if (currentPercent > percentComplete) { - infoPrint("connection %p is dumping out schema:%d%% of %s\n", - taos, currentPercent, stable); - percentComplete = currentPercent; + if (0 != avro_value_get_by_name( + &record, tmpBuf, &value, NULL)) { + errorPrint("%s() LN%d, avro_value_get_by_name(..%s..) failed\n", + __func__, __LINE__, + tbDes->cols[tbDes->columns + tag].field); } - } - - if (percentComplete < 100) { - errorPrint("%d%% - total %"PRId64" sub table(s) of stable: %s dumped\n", - percentComplete, tb, stable); - } else { - okPrint("total %"PRId64" sub table(s) of stable: %s dumped\n", - tb, stable); - } - - avro_value_iface_decref(wface); - freeRecordSchema(recordSchema); - avro_file_writer_close(db); - avro_schema_decref(schema); - tfree(jsonTagsSchema); + avro_value_t firsthalf, secondhalf; + uint8_t u8Temp = 0; + uint16_t u16Temp = 0; + uint32_t u32Temp = 0; + uint64_t u64Temp = 0; - return 0; -} + int type = tbDes->cols[tbDes->columns + tag].type; + switch (type) { + case TSDB_DATA_TYPE_BOOL: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + int tmp = atoi((const char *) + tbDes->cols[tbDes->columns+tag].value); + verbosePrint("%s() LN%d, before set_bool() tmp=%d\n", + __func__, __LINE__, (int)tmp); + avro_value_set_boolean(&branch, (tmp)?1:0); + } + break; -static int64_t dumpANormalTableBelongStb( - int64_t index, - TAOS *taos, - SDbInfo *dbInfo, char *stbName, - const TableDes *stbTableDes, - char *ntbName) { - int64_t count = 0; + case TSDB_DATA_TYPE_TINYINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + avro_value_set_int(&branch, + (int8_t)atoi((const char *) + tbDes->cols[tbDes->columns + + tag].value)); + } + break; - char dumpFilename[MAX_PATH_LEN] = {0}; - FILE *fp = NULL; + case TSDB_DATA_TYPE_SMALLINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + avro_value_set_int(&branch, + (int16_t)atoi((const char *) + tbDes->cols[tbDes->columns + + tag].value)); + } + break; - if (g_args.avro) { - if (0 != generateFilename(AVRO_TBTAGS, - dumpFilename, dbInfo, stbName, 0)) { - return -1; - } + case TSDB_DATA_TYPE_INT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + avro_value_set_int(&branch, + (int32_t)atoi((const char *) + tbDes->cols[tbDes->columns + + tag].value)); + } + break; + + case TSDB_DATA_TYPE_BIGINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + avro_value_set_long(&branch, + (int64_t)atoll((const char *) + tbDes->cols[tbDes->columns + tag].value)); + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + if (tbDes->cols[tbDes->columns + tag].var_value) { + avro_value_set_float(&branch, + atof(tbDes->cols[tbDes->columns + + tag].var_value)); + } else { + avro_value_set_float(&branch, + atof(tbDes->cols[tbDes->columns + + tag].value)); + } + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + if (tbDes->cols[tbDes->columns + tag].var_value) { + avro_value_set_double(&branch, + atof(tbDes->cols[tbDes->columns + + tag].var_value)); + } else { + avro_value_set_double(&branch, + atof(tbDes->cols[tbDes->columns + + tag].value)); + } + } + break; + + case TSDB_DATA_TYPE_BINARY: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + if (tbDes->cols[tbDes->columns + tag].var_value) { + avro_value_set_string(&branch, + tbDes->cols[tbDes->columns + + tag].var_value); + } else { + avro_value_set_string(&branch, + tbDes->cols[tbDes->columns + + tag].value); + } + } + break; + + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + if (tbDes->cols[tbDes->columns + tag].var_value) { + size_t nlen = strlen( + tbDes->cols[tbDes->columns + + tag].var_value); + char *bytes = malloc(nlen+1); + ASSERT(bytes); + + memcpy(bytes, + tbDes->cols[tbDes->columns + + tag].var_value, + nlen); + bytes[nlen] = 0; + avro_value_set_bytes(&branch, bytes, nlen); + free(bytes); + } else { + avro_value_set_bytes(&branch, + (void *)tbDes->cols[tbDes->columns + tag].value, + strlen(tbDes->cols[tbDes->columns + tag].value)); + } + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + avro_value_set_long(&branch, + (int64_t)atoll((const char *) + tbDes->cols[tbDes->columns + tag].value)); + } + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + u8Temp = (int8_t)atoi((const char *) + tbDes->cols[tbDes->columns + tag].value); + + int8_t n8tmp = (int8_t)(u8Temp - SCHAR_MAX); + avro_value_append(&branch, &firsthalf, NULL); + avro_value_set_int(&firsthalf, n8tmp); + debugPrint("%s() LN%d, first half is: %d, ", + __func__, __LINE__, n8tmp); + avro_value_append(&branch, &secondhalf, NULL); + avro_value_set_int(&secondhalf, (int32_t)SCHAR_MAX); + debugPrint("second half is: %d\n", SCHAR_MAX); + } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + u16Temp = (int16_t)atoi((const char *) + tbDes->cols[tbDes->columns + tag].value); + + int16_t n16tmp = (int16_t)(u16Temp - SHRT_MAX); + avro_value_append(&branch, &firsthalf, NULL); + avro_value_set_int(&firsthalf, n16tmp); + debugPrint("%s() LN%d, first half is: %d, ", + __func__, __LINE__, n16tmp); + avro_value_append(&branch, &secondhalf, NULL); + avro_value_set_int(&secondhalf, (int32_t)SHRT_MAX); + debugPrint("second half is: %d\n", SHRT_MAX); + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + u32Temp = (int32_t)atoi((const char *) + tbDes->cols[tbDes->columns + tag].value); + + int32_t n32tmp = (int32_t)(u32Temp - INT_MAX); + avro_value_append(&branch, &firsthalf, NULL); + avro_value_set_int(&firsthalf, n32tmp); + debugPrint("%s() LN%d, first half is: %d, ", + __func__, __LINE__, n32tmp); + avro_value_append(&branch, &secondhalf, NULL); + avro_value_set_int(&secondhalf, (int32_t)INT_MAX); + debugPrint("second half is: %d\n", INT_MAX); + } + + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (0 == strncmp( + tbDes->cols[tbDes->columns+tag].note, + "NUL", 3)) { + avro_value_set_branch(&value, 0, &branch); + avro_value_set_null(&branch); + } else { + avro_value_set_branch(&value, 1, &branch); + char *eptr; + u64Temp = strtoull((const char *) + tbDes->cols[tbDes->columns + tag].value, + &eptr, 10); + + int64_t n64tmp = (int64_t)(u64Temp - LONG_MAX); + avro_value_append(&branch, &firsthalf, NULL); + avro_value_set_long(&firsthalf, n64tmp); + debugPrint("%s() LN%d, first half is: %"PRId64", ", + __func__, __LINE__, n64tmp); + avro_value_append(&branch, &secondhalf, NULL); + avro_value_set_long(&secondhalf, (int64_t)LONG_MAX); + debugPrint("second half is: %"PRId64"\n", (int64_t)LONG_MAX); + } + + break; + + default: + errorPrint("Unknown type: %d\n", type); + break; + } + } + + if (0 != avro_file_writer_append_value(writer, &record)) { + errorPrint("%s() LN%d, Unable to write record to file. Message: %s\n", + __func__, __LINE__, + avro_strerror()); + } + avro_value_decref(&record); + + return 0; +} + +// open query with native or websocket +void* openQuery(void* taos , const char * sql) { +#ifdef WEBSOCKET + if (g_args.cloud || g_args.restful) { + WS_RES *ws_res = ws_query_timeout(taos, sql, g_args.ws_timeout); + int32_t code = ws_errno(ws_res); + if (code != 0) { + errorPrint("exe sql:%s failed. error code =%d\n", sql, code); + return NULL; + } + return ws_res; + } else { +#endif + TAOS_RES* res = taos_query(taos, sql); + int err = taos_errno(res); + if (err != 0) { + taos_free_result(res); + errorPrint("open query: %s execute failed. errcode=%d\n", sql, err); + return NULL; + } + return res; +#ifdef WEBSOCKET + } +#endif +} + +// close query and free result +void closeQuery(void* res) { +#ifdef WEBSOCKET + if (g_args.cloud || g_args.restful) { + if(res) { + ws_free_result(res); + } + return ; + } else { +#endif + if(res) { + taos_free_result(res); + } +#ifdef WEBSOCKET + } +#endif +} + +#ifdef WEBSOCKET +int readNextTableDesWS(void* ws_res, TableDes* tbDes, int *idx, int *cnt) { + // tbname, tagName , tagValue + int index = 0; + uint8_t type = 0; + uint32_t len = 0; + while( index < tbDes->tags) { + // get block + if(*idx >= *cnt || *cnt == 0) { + const void *data = NULL; + int ws_code = ws_fetch_block(ws_res, &data, cnt); + if (ws_code !=0 ) { + // read to end + errorPrint("read next ws_fetch_block failed, err code=%d idx=%d index=%d\n", ws_code, *idx, index); + return -1; + } + + if(*cnt == 0) { + infoPrint("read schema over. tag columns %d.\n", tbDes->tags); + break; + } + *idx = 0; + + } + + // read first column tbname + const void *val = ws_get_value_in_block(ws_res, *idx, 0, &type, &len); + if(val == NULL) { + errorPrint("read tbname failed, idx=%d cnt=%d \n", *idx, *cnt); + return -1; + } + + // tbname changed check + if(tbDes->name[0] == 0) { + // first set tbName + strncpy(tbDes->name, val, len); + } else { + // compare tbname change + if(!(strncmp(tbDes->name, val, len) == 0 + && tbDes->name[len] == 0)) { + // tbname cnanged, break + break; + } + } + + // read third column tagvalue + val = ws_get_value_in_block(ws_res, *idx, 2, &type, &len); + // copy tagvalue + if (NULL == val) { + strcpy(tbDes->cols[index].value, "NULL"); + strcpy(tbDes->cols[index].note , "NUL"); + } else if (0 != processFieldsValueV3(index, tbDes, val, len)) { + errorPrint("%s() LN%d, call processFieldsValueV3 tag_value: %p\n", + __func__, __LINE__, val); + return -1; + } + + // move next row + *idx = *idx + 1; + + // counter ++ + index++; + } + + // check tags count corrent + if(*cnt && index != tbDes->tags) { + errorPrint("child table %s read tags(%d) not equal stable tags (%d).\n", + tbDes->name, index, tbDes->tags); + return -1; + } + + return index; +} +#endif + + +// read next table tags to tbDes +int readNextTableDesNative(void* res, TableDes* tbDes) { + // tbname, tagName , tagValue + TAOS_ROW row; + int index = 0; + while( index < tbDes->tags && NULL != (row = taos_fetch_row(res))) { + // tbname changed check + int* lengths = taos_fetch_lengths(res); + if(tbDes->name[0] == 0) { + // first set tbName + strncpy(tbDes->name, row[0], lengths[0]); + } else { + // compare tbname change + if(!(strncmp(tbDes->name, row[0], lengths[0]) == 0 + && tbDes->name[lengths[0]] == 0)){ + // tbname cnanged, break + break; + } + } + + // copy tagname + if (NULL == row[2]) { + strcpy(tbDes->cols[index].value, "NULL"); + strcpy(tbDes->cols[index].note , "NUL"); + } else if (0 != processFieldsValueV3(index, tbDes, row[2], lengths[2])) { + errorPrint("%s() LN%d, call processFieldsValueV3 tag_value: %p\n", + __func__, __LINE__, row[1]); + return -1; + } + index++; + } + + // check tags count corrent + if(row && index != tbDes->tags) { + errorPrint("child table %s read tags(%d) not equal stable tags (%d).", + tbDes->name, index, tbDes->tags); + return -1; + } + + return index; +} + +#define SQL_LEN 512 +static int dumpStableMeta( + void *taos, + const SDbInfo *dbInfo, + TableDes *stbDes, + char **tbNameArr, + int64_t tbCount) { + // valid + char * stable = stbDes->name; + if (0 == stable[0]) { + errorPrint("%s() LN%d, pass wrong tbname\n", __func__, __LINE__); + return -1; + } + + // dump name + char dumpFilename[MAX_PATH_LEN] = {0}; + if (0 != generateFilename(AVRO_TBTAGS, dumpFilename, + dbInfo, stable, 0)) { + return -1; + } + debugPrint("%s() LN%d dumpFilename: %s\n", + __func__, __LINE__, dumpFilename); + + char *jsonTagsSchema = NULL; + if (0 != convertTbTagsDesToJsonWrap( + dbInfo->name, stable, stbDes, &jsonTagsSchema)) { + errorPrint("%s() LN%d, convertTbTagsDesToJsonWrap failed\n", + __func__, + __LINE__); + tfree(jsonTagsSchema); + return -1; + } + + // avro + debugPrint("tagsJson:\n%s\n", jsonTagsSchema); + avro_schema_t schema; + RecordSchema *recordSchema; + avro_file_writer_t avroWriter; + avro_value_iface_t *wface = prepareAvroWface( + dumpFilename, + jsonTagsSchema, &schema, &recordSchema, &avroWriter); + infoPrint("connection: %p is dumping out schema of sub-table(s) of %s \n", + taos, stable); + // free json + tfree(jsonTagsSchema); + + // query tag and values + char sql[SQL_LEN] = {0}; + snprintf(sql, SQL_LEN, + "select table_name,tag_name,tag_value from information_schema.ins_tags " + "where db_name='%s' and stable_name='%s';", + dbInfo->name, stable); + + void* tagsRes = openQuery(taos, sql); + if (tagsRes == NULL ) { + return -1; + } + +#ifdef WEBSOCKET + int idx = 0; + int cnt = 0; +#endif + + // loop read tables des + int size = sizeof(TableDes) + sizeof(ColDes) * stbDes->tags; + TableDes *tbDes = calloc(1, size); + int64_t tb = 0; + while (tb <= tbCount) { + // read tags + freeTbDes(tbDes, false); // free cols values + memset(tbDes->name, 0, sizeof(tbDes->name)); // reset zero + tbDes->tags = stbDes->tags; // stable tags same with child table + memcpy(tbDes->cols, &stbDes->cols[stbDes->columns], sizeof(ColDes)* stbDes->tags); // copy tag info + int ret; +#ifdef WEBSOCKET + if (g_args.cloud || g_args.restful) { + ret = readNextTableDesWS(tagsRes, tbDes, &idx, &cnt); + } else { +#endif + ret = readNextTableDesNative(tagsRes, tbDes); +#ifdef WEBSOCKET + } +#endif + + if(ret < 0){ + // read error + freeTbDes(tbDes, true); + return ret; + } else if (ret == 0) { + // read end , break + break; + } + + // dump tbname to array + tbNameArr[tb] = strdup(tbDes->name); + + // write tags to avro + ret = writeTagsToAvro( + dbInfo->name, + stbDes, tbDes, + avroWriter, wface); + if(ret < 0) { + // write error + freeTbDes(tbDes, true); + return ret; + } + + // sucess print + tb++; + infoPrint("connection %p is dumping out schema: %"PRId64" from %s.%s\n", taos, tb, stable, tbDes->name); + } + okPrint("total %"PRId64" table(s) of stable: %s schema dumped.\n", tb, stable); + + // free + closeQuery(tagsRes); + avro_value_iface_decref(wface); + freeRecordSchema(recordSchema); + avro_file_writer_close(avroWriter); + avro_schema_decref(schema); + freeTbDes(tbDes, true); + + return 0; +} + +static int64_t dumpANormalTableBelongStb( + int64_t index, + TAOS *taos, + SDbInfo *dbInfo, char *stbName, + const TableDes *stbTableDes, + char *ntbName) { + int64_t count = 0; + + char dumpFilename[MAX_PATH_LEN] = {0}; + FILE *fp = NULL; + + if (g_args.avro) { + if (0 != generateFilename(AVRO_TBTAGS, + dumpFilename, dbInfo, stbName, 0)) { + return -1; + } debugPrint("%s() LN%d dumpFilename: %s\n", __func__, __LINE__, dumpFilename); @@ -10659,10 +11211,7 @@ static void dumpNormalTablesOfStbWS( char *dumpFilename) { for (int64_t i = pThreadInfo->from; i < (pThreadInfo->from + pThreadInfo->count); i++ ) { - char tbName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(tbName, - pThreadInfo->tbNameArr + i * TSDB_TABLE_NAME_LEN, - TSDB_TABLE_NAME_LEN); + char* tbName = pThreadInfo->tbNameArr[i]; debugPrint("%s() LN%d, [%d] sub table %"PRId64": name: %s\n", __func__, __LINE__, pThreadInfo->threadIndex, i, @@ -10676,7 +11225,7 @@ static void dumpNormalTablesOfStbWS( pThreadInfo->dbInfo, true, pThreadInfo->stbName, - pThreadInfo->stbTableDes, + pThreadInfo->stbDes, tbName, pThreadInfo->precision, dumpFilename, @@ -10688,13 +11237,17 @@ static void dumpNormalTablesOfStbWS( pThreadInfo->dbInfo, true, pThreadInfo->stbName, - pThreadInfo->stbTableDes, + pThreadInfo->stbDes, tbName, pThreadInfo->precision, NULL, fp); } + // show progress + atomic_add_fetch_64(&g_tableDone, 1); + infoPrint("%s.%s %"PRId64"/%"PRId64" %s dump data ok.\n", + g_dbName, g_stbName, g_tableDone, g_tableCount, tbName); if (count < 0) { break; } else { @@ -10706,16 +11259,13 @@ static void dumpNormalTablesOfStbWS( } #endif -static void dumpNormalTablesOfStbNative( +static void dumpTablesOfStbNative( threadInfo *pThreadInfo, FILE *fp, char *dumpFilename) { for (int64_t i = pThreadInfo->from; i < pThreadInfo->from + pThreadInfo->count; i++) { - char tbName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(tbName, - pThreadInfo->tbNameArr + i * TSDB_TABLE_NAME_LEN, - TSDB_TABLE_NAME_LEN); + char* tbName = pThreadInfo->tbNameArr[i]; debugPrint("%s() LN%d, [%d] sub table %"PRId64": name: %s\n", __func__, __LINE__, pThreadInfo->threadIndex, i, tbName); @@ -10728,7 +11278,7 @@ static void dumpNormalTablesOfStbNative( pThreadInfo->dbInfo, true, pThreadInfo->stbName, - pThreadInfo->stbTableDes, + pThreadInfo->stbDes, tbName, pThreadInfo->precision, dumpFilename, @@ -10740,7 +11290,7 @@ static void dumpNormalTablesOfStbNative( pThreadInfo->dbInfo, true, pThreadInfo->stbName, - pThreadInfo->stbTableDes, + pThreadInfo->stbDes, tbName, pThreadInfo->precision, NULL, @@ -10760,7 +11310,7 @@ static void dumpNormalTablesOfStbNative( return; } -static void *dumpNormalTablesOfStb(void *arg) { +static void *dumpTablesOfStbThread(void *arg) { threadInfo *pThreadInfo = (threadInfo *)arg; debugPrint("dump table from = \t%"PRId64"\n", pThreadInfo->from); @@ -10798,7 +11348,7 @@ static void *dumpNormalTablesOfStb(void *arg) { dumpNormalTablesOfStbWS(pThreadInfo, fp, dumpFilename); } else { #endif - dumpNormalTablesOfStbNative(pThreadInfo, fp, dumpFilename); + dumpTablesOfStbNative(pThreadInfo, fp, dumpFilename); #ifdef WEBSOCKET } #endif @@ -10809,120 +11359,16 @@ static void *dumpNormalTablesOfStb(void *arg) { return NULL; } -static int64_t dumpNtbOfStbByThreads( - void *taos_v, - SDbInfo *dbInfo, - const char *stbName) { - int64_t ntbCount; - char *command = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN); - if (NULL == command) { - errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); - return -1; - } - - if (3 == g_majorVersionOfClient) { - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - g_args.db_escape_char - ? "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " - "from `%s`.%s%s%s)" - : "SELECT COUNT(*) FROM (SELECT DISTINCT(TBNAME) " - "from %s.%s%s%s)", - dbInfo->name, g_escapeChar, stbName, g_escapeChar); - } else { - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - g_args.db_escape_char - ? "SELECT COUNT(TBNAME) FROM `%s`.%s%s%s" - : "SELECT COUNT(TBNAME) FROM %s.%s%s%s", - dbInfo->name, g_escapeChar, stbName, g_escapeChar); - } - - infoPrint("Getting tables' number of super table (%s) ...\n", stbName); - -#ifdef WEBSOCKET - if (g_args.cloud || g_args.restful) { - ntbCount = getNtbCountOfStbWS(command); - } else { -#endif - ntbCount = getNtbCountOfStbNative(dbInfo->name, stbName, command); -#ifdef WEBSOCKET - } -#endif - - // set progress to global - g_tableCount = ntbCount; - g_tableDone = 0; - strcpy(g_dbName, dbInfo->name); - strcpy(g_stbName, stbName); - - infoPrint("%s() LN%d, %s's %s's total normal table count: %"PRId64"\n", - __func__, __LINE__, dbInfo->name, stbName, ntbCount); - if (ntbCount <= 0) { - return 0; - } - - char *tbNameArr = NULL; - - TableDes *stbTableDes = (TableDes *)calloc(1, sizeof(TableDes) - + sizeof(ColDes) * TSDB_MAX_COLUMNS); - if (NULL == stbTableDes) { - errorPrint("%s() LN%d, memory allocation failed!\n", - __func__, __LINE__); - return -1; - } - - int colCount = 0; - colCount = colCount; -#ifdef WEBSOCKET - if (g_args.cloud || g_args.restful) { - colCount = getTableDesWS(taos_v, dbInfo->name, - stbName, stbTableDes, true); - } else { -#endif - colCount = getTableDesNative(taos_v, dbInfo->name, - stbName, stbTableDes, true); -#ifdef WEBSOCKET - } -#endif - if (colCount < 0) { - errorPrint("%s() LN%d, failed to get stable[%s] schema\n", - __func__, __LINE__, stbName); - if (stbTableDes) { - freeTbDes(stbTableDes); - } - exit(-1); - } - - if (g_args.avro) { - int ret = createMTableAvroHead( - taos_v, - dbInfo, - stbName, - &stbTableDes, - &tbNameArr); - if (-1 == ret) { - errorPrint("%s() LN%d, failed to dump table\n", - __func__, __LINE__); - if (tbNameArr) { - free(tbNameArr); - } - freeTbDes(stbTableDes); - return -1; - } - } else { - fillTbNameArr( - taos_v, &tbNameArr, dbInfo, stbName); - } - +int dumpSTableData(SDbInfo* dbInfo, TableDes* stbDes, char** tbNameArr, int64_t tbCount) { int threads = g_args.thread_num; - - int64_t a = ntbCount / threads; - if (a < 1) { - threads = ntbCount; - a = 1; + int64_t batch = tbCount / threads; + if (batch < 1) { + threads = tbCount; + batch = 1; } ASSERT(threads); - int64_t b = ntbCount % threads; + int64_t mod = tbCount % threads; pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); @@ -10941,10 +11387,6 @@ static int64_t dumpNtbOfStbByThreads( __func__, __LINE__, ws_errstr(NULL)); - if (tbNameArr) { - free(tbNameArr); - } - freeTbDes(stbTableDes); free(pids); free(infos); return -1; @@ -10952,10 +11394,6 @@ static int64_t dumpNtbOfStbByThreads( } else { #endif // WEBSOCKET if (NULL == (pThreadInfo->taos = taosConnect(dbInfo->name))) { - if (tbNameArr) { - free(tbNameArr); - } - freeTbDes(stbTableDes); free(pids); free(infos); return -1; @@ -10965,7 +11403,7 @@ static int64_t dumpNtbOfStbByThreads( #endif pThreadInfo->threadIndex = i; - pThreadInfo->count = (i < b)?a+1:a; + pThreadInfo->count = (i < mod) ? batch+1 : batch; pThreadInfo->from = (i == 0)?0: ((threadInfo *)(infos + i - 1))->from + ((threadInfo *)(infos + i - 1))->count; @@ -10976,11 +11414,11 @@ static int64_t dumpNtbOfStbByThreads( exit(EXIT_FAILURE); } - strcpy(pThreadInfo->stbName, stbName); - pThreadInfo->stbTableDes = stbTableDes; - pThreadInfo->tbNameArr = tbNameArr; + strcpy(pThreadInfo->stbName, stbDes->name); + pThreadInfo->stbDes = stbDes; + pThreadInfo->tbNameArr = tbNameArr; if (pthread_create(pids + i, NULL, - dumpNormalTablesOfStb, pThreadInfo) != 0) { + dumpTablesOfStbThread, pThreadInfo) != 0) { errorPrint("%s() LN%d, thread[%d] failed to start. " "The errno is %d. Reason: %s\n", __func__, __LINE__, @@ -10999,22 +11437,141 @@ static int64_t dumpNtbOfStbByThreads( } } - infoPrint("%s\n","close taos connections..."); + infoPrint("super table (%s) dump %"PRId64" child data ok. close taos connections...\n", + stbDes->name, tbCount); for (int32_t i = 0; i < threads; i++) { pThreadInfo = infos + i; taos_close(pThreadInfo->taos); } - if (tbNameArr) { - free(tbNameArr); - } - - freeTbDes(stbTableDes); free(pids); free(infos); return 0; } +// free names +void freeTbNameArr(char ** tbNameArr, int64_t tbCount) { + for (int64_t i = 0; i < tbCount; i++) { + if (tbNameArr[i]) { + free(tbNameArr[i]); + } + } + free(tbNameArr); +} + +// dump stable meta and data by threads +static int64_t dumpStable( + void *taos_v, + SDbInfo *dbInfo, + const char *stbName) { + // show progress + int ret = -1; + infoPrint("start dump out super table data (%s) ...\n", stbName); + + // + // get super table meta + // + + // malloc stable des + TableDes *stbDes = (TableDes *)calloc(1, sizeof(TableDes) + + sizeof(ColDes) * TSDB_MAX_COLUMNS); + if (NULL == stbDes) { + errorPrint("%s() LN%d, memory allocation failed!\n", + __func__, __LINE__); + return -1; + } + + // obtain stable des data + int colCount = 0; +#ifdef WEBSOCKET + if (g_args.cloud || g_args.restful) { + colCount = getTableDesWS(taos_v, dbInfo->name, + stbName, stbDes, true); + } else { +#endif + colCount = getTableDesNative(taos_v, dbInfo->name, + stbName, stbDes, true); +#ifdef WEBSOCKET + } +#endif + if (colCount < 0) { + errorPrint("%s() LN%d, failed to get stable[%s] schema\n", + __func__, __LINE__, stbName); + freeTbDes(stbDes, true); + exit(-1); + } + // show progress + infoPrint("start dump super table meta (%s) col:%d tags:%d ...\n", + stbName, stbDes->columns, stbDes->tags); + + // get stable child count + int64_t tbCount = 0; +#ifdef WEBSOCKET + if (g_args.cloud || g_args.restful) { + tbCount = getNtbCountOfStbWS(dbInfo->name, stbName); + } else { +#endif + tbCount = getTbCountOfStbNative(dbInfo->name, stbName); +#ifdef WEBSOCKET + } +#endif + if(tbCount < 0 ) { + errorPrint("get stable %s failed.", stbName); + freeTbDes(stbDes, true); + exit(-1); + } + // show progress + infoPrint("The number of tables of %s is %"PRId64"!\n", stbName, tbCount); + // set progress to global + g_tableCount = tbCount; + g_tableDone = 0; + strcpy(g_dbName, dbInfo->name); + strcpy(g_stbName, stbName); + + // + // dump meta + // + char** tbNameArr = (char**)calloc(tbCount, sizeof(char*)); + if (g_args.avro) { + ret = dumpStableMeta( + taos_v, + dbInfo, + stbDes, + tbNameArr, + tbCount); + if (-1 == ret) { + errorPrint("%s() LN%d, failed to dump table\n", + __func__, __LINE__); + freeTbNameArr(tbNameArr, tbCount); + freeTbDes(stbDes, true); + return -1; + } + } else { + fillTbNameArr(taos_v, tbNameArr, dbInfo, stbName, tbCount); + } + + if(tbCount <= 0) { + freeTbNameArr(tbNameArr, tbCount); + freeTbDes(stbDes, true); + + if (tbCount == 0) { + infoPrint("super table (%s) no child table, skip dump out.\n", stbName); + return 0; + } else { + infoPrint("super table (%s) get child count failed.\n", stbName); + return -1; + } + } + + // + // dump data + // + ret = dumpSTableData(dbInfo, stbDes, tbNameArr, tbCount); + freeTbNameArr(tbNameArr, tbCount); + freeTbDes(stbDes, true); + return ret; +} + static int64_t dumpStbAndChildTb( void *taos_v, SDbInfo *dbInfo, const char *stable, FILE *fpDbs) { int64_t ret = 0; @@ -11038,7 +11595,7 @@ static int64_t dumpStbAndChildTb( fpDbs); if (ret >= 0) { - ret = dumpNtbOfStbByThreads( + ret = dumpStable( taos_v, dbInfo, stable); @@ -11048,7 +11605,7 @@ static int64_t dumpStbAndChildTb( stable); } - freeTbDes(stbTableDes); + freeTbDes(stbTableDes, true); return ret; } @@ -12535,7 +13092,7 @@ static int dumpOut() { __func__, __LINE__, tableRecordInfo.tableRecord.stable); } - freeTbDes(stbTableDes); + freeTbDes(stbTableDes, true); } else { ret = dumpANormalTableNotBelong( i, diff --git a/src/toolstime.c b/src/toolstime.c index 825131ae..305cd074 100644 --- a/src/toolstime.c +++ b/src/toolstime.c @@ -814,3 +814,99 @@ FORCE_INLINE int64_t toolsGetTimestampNs() { } FORCE_INLINE void toolsMsleep(int32_t mseconds) { usleep(mseconds * 1000); } + +struct tm *tLocalTime(const time_t *timep, struct tm *result, char *buf) { + struct tm *res = NULL; + if (timep == NULL) { + return NULL; + } + if (result == NULL) { + res = localtime(timep); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } + return res; + } +#ifdef WINDOWS + if (*timep < -2208988800LL) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + + SYSTEMTIME s; + FILETIME f; + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + if (localtime_s(&tm1, &tt) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + offset.QuadPart = ((uint64_t)116445024000000000ULL); + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth - 1; + result->tm_year = s.wYear - 1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; +#else + res = localtime_r(timep, result); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } +#endif + return result; +} + +char *toolsFormatTimestamp(char *buf, int64_t val, int32_t precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm ptm = {0}; + if (tLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} diff --git a/tests/taosbenchmark/.DS_Store b/tests/taosbenchmark/.DS_Store new file mode 100644 index 00000000..68a356aa Binary files /dev/null and b/tests/taosbenchmark/.DS_Store differ diff --git a/tests/taosbenchmark/commandline-sml.py b/tests/taosbenchmark/commandline-sml.py index 436720a4..aebe88cb 100644 --- a/tests/taosbenchmark/commandline-sml.py +++ b/tests/taosbenchmark/commandline-sml.py @@ -88,9 +88,9 @@ def run(self): tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 1) - cmd = "%s -N -I sml -y" % binPath + cmd = "%s -I sml -t 10 -n 10000 -y" % binPath tdLog.info("%s" % cmd) - assert os.system("%s" % cmd) != 0 + tdSql.checkData(0, 0, 10*10000) def stop(self): tdSql.close() diff --git a/tests/taosbenchmark/commandline.py b/tests/taosbenchmark/commandline.py index 7e03b0c9..fe7b5cae 100644 --- a/tests/taosbenchmark/commandline.py +++ b/tests/taosbenchmark/commandline.py @@ -338,7 +338,7 @@ def run(self): tdLog.info("%s" % cmd) assert os.system("%s" % cmd) != 0 - cmd = "%s -n 1 -t 1 -y -A int,json" % binPath + cmd = "%s -n 1 -t 1 -y -A json" % binPath tdLog.info("%s" % cmd) assert os.system("%s" % cmd) != 0 diff --git a/tests/taosbenchmark/json/insert-sample-ts-stmt.json b/tests/taosbenchmark/json/insert-sample-ts-stmt.json index f431325a..b4bf21b7 100644 --- a/tests/taosbenchmark/json/insert-sample-ts-stmt.json +++ b/tests/taosbenchmark/json/insert-sample-ts-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 1, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/taosbenchmark/json/stmt_sample_doesnt_use_ts.json b/tests/taosbenchmark/json/stmt_sample_doesnt_use_ts.json index d7f09388..67253fef 100644 --- a/tests/taosbenchmark/json/stmt_sample_doesnt_use_ts.json +++ b/tests/taosbenchmark/json/stmt_sample_doesnt_use_ts.json @@ -6,6 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 1, + "create_table_thread_count": 2, "connection_pool_size": 20, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", diff --git a/tests/taosbenchmark/json/stmt_sample_use_ts-subtable.json b/tests/taosbenchmark/json/stmt_sample_use_ts-subtable.json index 08d283b5..67fce728 100644 --- a/tests/taosbenchmark/json/stmt_sample_use_ts-subtable.json +++ b/tests/taosbenchmark/json/stmt_sample_use_ts-subtable.json @@ -6,6 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, + "create_table_thread_count": 2, "connection_pool_size": 20, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", diff --git a/tests/taosbenchmark/json/stmt_sample_use_ts.json b/tests/taosbenchmark/json/stmt_sample_use_ts.json index 5d9aeefb..3a6185ef 100644 --- a/tests/taosbenchmark/json/stmt_sample_use_ts.json +++ b/tests/taosbenchmark/json/stmt_sample_use_ts.json @@ -6,6 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 1, + "create_table_thread_count": 2, "connection_pool_size": 20, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", diff --git a/tests/taosbenchmark/json/taosc_sample_use_ts-subtable.json b/tests/taosbenchmark/json/taosc_sample_use_ts-subtable.json index b63a1b39..e2396a99 100644 --- a/tests/taosbenchmark/json/taosc_sample_use_ts-subtable.json +++ b/tests/taosbenchmark/json/taosc_sample_use_ts-subtable.json @@ -6,6 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, + "create_table_thread_count": 2, "connection_pool_size": 20, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", diff --git a/tests/taosbenchmark/json/taosc_sample_use_ts.json b/tests/taosbenchmark/json/taosc_sample_use_ts.json index 1ff0cf2d..109e0428 100644 --- a/tests/taosbenchmark/json/taosc_sample_use_ts.json +++ b/tests/taosbenchmark/json/taosc_sample_use_ts.json @@ -6,6 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, + "create_table_thread_count": 2, "connection_pool_size": 20, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", diff --git a/tests/taosbenchmark/sml_telnet_alltypes.py b/tests/taosbenchmark/sml_telnet_alltypes.py index 2536763c..cea2fa1f 100644 --- a/tests/taosbenchmark/sml_telnet_alltypes.py +++ b/tests/taosbenchmark/sml_telnet_alltypes.py @@ -93,7 +93,7 @@ def run(self): if major_ver == "3": tdSql.checkData(1, 1, "VARCHAR") tdSql.checkData( - 1, 2, 16 + 1, 2, 8 ) # 3.0 will use a bit more space for schemaless create table else: tdSql.checkData(1, 1, "BINARY") @@ -102,7 +102,7 @@ def run(self): tdSql.query("describe db.stb13") tdSql.checkData(1, 1, "NCHAR") if major_ver == "3": - tdSql.checkData(1, 2, 16) + tdSql.checkData(1, 2, 8) else: tdSql.checkData(1, 2, 8) diff --git a/tests/taosbenchmark/v3/cloud-test.py b/tests/taosbenchmark/v3/cloud-test.py index b10a5a12..01a9fbf9 100644 --- a/tests/taosbenchmark/v3/cloud-test.py +++ b/tests/taosbenchmark/v3/cloud-test.py @@ -59,10 +59,11 @@ def getPath(self, tool="taosBenchmark"): def run(self): binPath = self.getPath() - cmd = "%s -T 1 -t 2 -n 10 -g -y" % binPath + cmd = "%s -T 1 -t 2 -n 10 -y" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) + ''' taosPath = self.getPath("taos") cmd = f"{taosPath} -s 'select count(*) from test.meters'" tdLog.info(f"{cmd}") @@ -72,6 +73,7 @@ def run(self): tdLog.info("count of records is correct!") else: tdLog.exit("count of records is incorrect") + ''' def stop(self): tdSql.close()