diff --git a/ansible/create_cluster.yml b/ansible/create_cluster.yml index 29246a410..461ab543b 100644 --- a/ansible/create_cluster.yml +++ b/ansible/create_cluster.yml @@ -10,8 +10,7 @@ # limitations under the License. --- -- name: Clean Pre Macrobench - when: not clean_previous_exec | bool +- name: Clean Post Macrobench import_playbook: clean_macrobench.yml - name: Build Vitess @@ -70,3 +69,6 @@ - name: Apply VSchema shell: | vtctlclient --server {{ groups['vtctld'][0] }}:15999 ApplyVSchema -- --vschema="$(cat /tmp/vschema_sysbench.json)" main + + - name: Print go version + debug: msg="VSchema is applied." \ No newline at end of file diff --git a/ansible/macrobench.yml b/ansible/macrobench.yml index a9aa1c52c..5ed71c749 100644 --- a/ansible/macrobench.yml +++ b/ansible/macrobench.yml @@ -43,7 +43,7 @@ gateway_targets: "{% set targets = [] -%}{% for host in groups['vtgate'] -%}{% for gateway in hostvars[host]['gateways'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (gateway.port | default(15001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}" vtctld_targets: "{% set targets = [] -%}{% for host in groups['vtctld'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (vtctld_port | default(15000) | string )) }}{% endfor -%}{{ targets }}" node_targets: "{% set targets= [] -%}{% for host in groups['all'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':9100' ) }}{% endfor -%}{{ targets }}" - prometheus_skip_install: true + prometheus_skip_install: false prometheus_version: 2.49.1 prometheus_global: scrape_interval: 15s diff --git a/ansible/roles/etcd/tasks/main.yml b/ansible/roles/etcd/tasks/main.yml index b5a2d39c9..20544eb42 100644 --- a/ansible/roles/etcd/tasks/main.yml +++ b/ansible/roles/etcd/tasks/main.yml @@ -10,6 +10,8 @@ # limitations under the License. --- +- include_tasks: clean.yml + - include_tasks: install.yml when: not etcd_launch | bool diff --git a/ansible/roles/macrobench/tasks/main.yml b/ansible/roles/macrobench/tasks/main.yml index 187c8fcaf..ea0001560 100644 --- a/ansible/roles/macrobench/tasks/main.yml +++ b/ansible/roles/macrobench/tasks/main.yml @@ -14,6 +14,11 @@ become: yes become_user: root block: + - name: Remove old arewefastyet + shell: | + rm -Rf /go/src/github.com/vitessio/arewefastyet + changed_when: false + - name: git clone arewefastyet git: repo: "{{ arewefastyet_git_repo }}" diff --git a/ansible/roles/sysbench/tasks/install.yml b/ansible/roles/sysbench/tasks/install.yml index b75b62624..069143a6e 100644 --- a/ansible/roles/sysbench/tasks/install.yml +++ b/ansible/roles/sysbench/tasks/install.yml @@ -10,36 +10,33 @@ # limitations under the License. --- -- name: Install Sysbench - when: install_sysbench is defined - block: - - name: Update Sysbench - become: yes - become_user: root - git: - repo: https://github.com/planetscale/sysbench - dest: /src/sysbench - version: master - depth: 1 - force: 1 +- name: Update Sysbench + become: yes + become_user: root + git: + repo: https://github.com/planetscale/sysbench + dest: /src/sysbench + version: master + depth: 1 + force: 1 - - name: Build sysbench - become: yes - become_user: root - shell: | - cd /src/sysbench - ./autogen.sh - ./configure - make -j - make install - changed_when: false +- name: Build sysbench + become: yes + become_user: root + shell: | + cd /src/sysbench + ./autogen.sh + ./configure + make -j + make install + changed_when: false - - name: Get tpcc - become: yes - become_user: root - git: - repo: https://github.com/planetscale/sysbench-tpcc - dest: /src/sysbench-tpcc - version: master - depth: 1 - force: 1 +- name: Get tpcc + become: yes + become_user: root + git: + repo: https://github.com/planetscale/sysbench-tpcc + dest: /src/sysbench-tpcc + version: master + depth: 1 + force: 1 diff --git a/ansible/roles/vitess_build/tasks/main.yml b/ansible/roles/vitess_build/tasks/main.yml index b7fe13af5..9cd85effd 100644 --- a/ansible/roles/vitess_build/tasks/main.yml +++ b/ansible/roles/vitess_build/tasks/main.yml @@ -10,19 +10,12 @@ # limitations under the License. --- -- name: ensure vitess tmp directory - become: yes - become_user: root - file: - state: directory - path: '{{ vitess_root }}/tmp' - owner: '{{ vitess_user }}' - group: '{{ vitess_group }}' +- name: Stop existing Vitess to build a new one + include_tasks: clean.yml - name: Update Vitess become: yes become_user: root - when: commit_previous_exec != vitess_git_version or not clean_previous_exec | bool block: - name: Fetch Updated Vitess include_tasks: install_vitess.yml @@ -62,7 +55,6 @@ changed_when: false - name: Disbale AppArmor - when: not clean_previous_exec | bool block: - name: link apparmor file: diff --git a/ansible/roles/vtctld/tasks/main.yml b/ansible/roles/vtctld/tasks/main.yml index 5237b8332..619dcb66f 100644 --- a/ansible/roles/vtctld/tasks/main.yml +++ b/ansible/roles/vtctld/tasks/main.yml @@ -10,12 +10,18 @@ # limitations under the License. --- +- name: Create users + include_role: + name: "vitess_build" + tasks_from: create_users + - name: make sure vtctld is stopped when: clean_vtctld | bool include_role: name: "vtctld" tasks_from: "clean" + - name: ensure vitess directories become: yes become_user: root diff --git a/ansible/roles/vtgate/tasks/main.yml b/ansible/roles/vtgate/tasks/main.yml index db015f3f7..9811115a7 100644 --- a/ansible/roles/vtgate/tasks/main.yml +++ b/ansible/roles/vtgate/tasks/main.yml @@ -16,6 +16,11 @@ name: "vtgate" tasks_from: clean +- name: Create users + include_role: + name: "vitess_build" + tasks_from: create_users + - name: ensure vitess directories become: yes become_user: root diff --git a/ansible/roles/vttablet/tasks/main.yml b/ansible/roles/vttablet/tasks/main.yml index 94a140e2c..1513910c0 100644 --- a/ansible/roles/vttablet/tasks/main.yml +++ b/ansible/roles/vttablet/tasks/main.yml @@ -16,6 +16,11 @@ name: "vttablet" tasks_from: "clean" +- name: Create users + include_role: + name: "vitess_build" + tasks_from: create_users + - name: ensure vitess directories become: yes become_user: root @@ -33,9 +38,8 @@ - name: install systemd units include_tasks: systemd.yml -- name: Install config files +- name: install config files include_tasks: vttablet.yml loop: '{{ tablets }}' loop_control: loop_var: tablet - diff --git a/go/exec/exec.go b/go/exec/exec.go index 0d1794a52..252603ffe 100644 --- a/go/exec/exec.go +++ b/go/exec/exec.go @@ -141,12 +141,6 @@ type Exec struct { vitessConfig vitessConfig vitessSchemaPath string - - // We always fetch the previous execution to let Ansible skip certain steps of the benchmark - // execution. Like cleaning up at the start of the execution, this is not required when the - // previous execution was clean and did not fail. Moreover, we can skip the fetch and build - // of Vitess if the previous execution share the same commit as the current execution. - previousExecution *Exec } const ( @@ -261,13 +255,6 @@ func (e *Exec) Prepare() error { return err } - // get previous benchmark - previousExec, err := getPreviousExecution(e.clientDB) - if err != nil { - return err - } - e.previousExecution = previousExec - // insert new exec in SQL if _, err = e.clientDB.Insert( "INSERT INTO execution(uuid, status, source, git_ref, type, pull_nb, go_version) VALUES(?, ?, ?, ?, ?, ?, ?)", @@ -393,10 +380,6 @@ func (e *Exec) prepareAnsibleForExecution() error { // runtime related values e.AnsibleConfig.AddExtraVar(ansible.KeyGoVersion, e.GolangVersion) - // previous execution - e.AnsibleConfig.AddExtraVar(ansible.KeyCleanPreviousExec, e.previousExecution.Status == StatusFinished) - e.AnsibleConfig.AddExtraVar(ansible.KeyCommitPreviousExec, e.previousExecution.GitRef) - // stats database related values e.statsRemoteDBConfig.AddToAnsible(&e.AnsibleConfig) return nil @@ -438,29 +421,6 @@ func (e *Exec) defineVersionNameOfVitess() error { return nil } -func getPreviousExecution(client storage.SQLClient) (*Exec, error) { - query := "SELECT uuid, status, git_ref, started_at, finished_at, source, type, pull_nb, go_version FROM execution ORDER BY started_at DESC LIMIT 1" - result, err := client.Select(query) - if err != nil { - return nil, err - } - defer result.Close() - - var eUUID string - exec := &Exec{} - if result.Next() { - err = result.Scan(&eUUID, &exec.Status, &exec.GitRef, &exec.StartedAt, &exec.FinishedAt, &exec.Source, &exec.TypeOf, &exec.PullNB, &exec.GolangVersion) - if err != nil { - return nil, err - } - exec.UUID, err = uuid.Parse(eUUID) - if err != nil { - return nil, err - } - } - return exec, nil -} - func GetRecentExecutions(client storage.SQLClient) ([]*Exec, error) { var res []*Exec query := "SELECT uuid, status, git_ref, started_at, finished_at, source, type, pull_nb, go_version FROM execution ORDER BY started_at DESC LIMIT 50" @@ -564,6 +524,23 @@ func GetPreviousFromSourceMacrobenchmark(client storage.SQLClient, source, typeO return } +// GetLatestDailyJobForMicrobenchmarks will fetch and return the commit sha for which +// the last daily job for microbenchmarks was run +func GetLatestDailyJobForMicrobenchmarks(client storage.SQLClient) (gitSha string, err error) { + query := "select git_ref from execution where source = \"cron\" and status = \"finished\" and type = \"micro\" order by started_at desc limit 1" + rows, err := client.Select(query) + if err != nil { + return "", err + } + + defer rows.Close() + for rows.Next() { + err = rows.Scan(&gitSha) + return gitSha, err + } + return "", nil +} + // GetLatestDailyJobForMacrobenchmarks will fetch and return the commit sha for which // the last daily job for macrobenchmarks was run func GetLatestDailyJobForMacrobenchmarks(client storage.SQLClient) (gitSha string, err error) { @@ -601,6 +578,45 @@ func ExistsMacrobenchmark(client storage.SQLClient, gitRef, source, typeOf, stat return result.Next(), nil } +func ExistsMacrobenchmarkStartedToday(client storage.SQLClient, gitRef, source, typeOf, planner, status string) (bool, error) { + query := fmt.Sprintf("SELECT uuid FROM execution e, macrobenchmark m WHERE e.status = '%s' AND e.git_ref = ? AND e.type = ? AND e.source = ? AND m.vtgate_planner_version = ? AND e.uuid = m.exec_uuid AND e.started_at >= CURDATE()", status) + result, err := client.Select(query, gitRef, typeOf, source, planner) + if err != nil { + return false, err + } + exists := result.Next() + result.Close() + if exists { + return true, nil + } + query = fmt.Sprintf("SELECT uuid FROM execution e WHERE e.status = '%s' AND e.git_ref = ? AND e.type = ? AND e.source = ? AND e.started_at >= CURDATE()", status) + result, err = client.Select(query, gitRef, typeOf, source) + if err != nil { + return false, err + } + defer result.Close() + for result.Next() { + var exec_uuid string + err = result.Scan(&exec_uuid) + if err != nil { + return false, err + } + query = "SELECT exec_uuid FROM macrobenchmark m WHERE m.exec_uuid = ?" + resultMacro, err := client.Select(query, exec_uuid) + if err != nil { + return false, err + } + defer resultMacro.Close() + + next := resultMacro.Next() + resultMacro.Close() + if !next { + return true, nil + } + } + return false, nil +} + func DeleteExecution(client storage.SQLClient, gitRef, UUID, source string) error { query := fmt.Sprintf("DELETE FROM execution WHERE uuid LIKE '%%%s%%' AND git_ref LIKE '%%%s%%' AND source = '%s'", UUID, gitRef, source) _, err := client.Select(query) @@ -608,4 +624,4 @@ func DeleteExecution(client storage.SQLClient, gitRef, UUID, source string) erro return err } return nil -} +} \ No newline at end of file diff --git a/go/infra/ansible/vars.go b/go/infra/ansible/vars.go index 3b4d91b92..dcda54074 100644 --- a/go/infra/ansible/vars.go +++ b/go/infra/ansible/vars.go @@ -89,15 +89,6 @@ const ( // the execution of the benchmark. KeyGoVersion = "golang_gover" - // Previous execution keys - - // KeyCleanPreviousExec tells whether the previous execution was a clean benchmark or not - // this is useful when doing optimization to skip certain steps of the setup - KeyCleanPreviousExec = "clean_previous_exec" - - // KeyCommitPreviousExec is the commit SHA of the previous execution. - KeyCommitPreviousExec = "commit_previous_exec" - // Stats database related keys // KeyStatsDBHost corresponding value in the map is the hostname for the stats