Merge pull request #512 from vitessio/revert-latest-improvements

Revert latest Ansible improvement

ansible/create_cluster.yml

@@ -10,8 +10,7 @@
 # limitations under the License.
 
 ---
-- name: Clean Pre Macrobench
-  when: not clean_previous_exec | bool
+- name: Clean Post Macrobench
   import_playbook: clean_macrobench.yml
 
 - name: Build Vitess
@@ -70,3 +69,6 @@
     - name: Apply VSchema
       shell: |
         vtctlclient --server {{ groups['vtctld'][0] }}:15999 ApplyVSchema -- --vschema="$(cat /tmp/vschema_sysbench.json)" main
+
+    - name: Print go version
+      debug: msg="VSchema is applied."

ansible/macrobench.yml

@@ -43,7 +43,7 @@
     gateway_targets: "{% set targets = [] -%}{% for host in groups['vtgate'] -%}{% for gateway in hostvars[host]['gateways'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (gateway.port | default(15001) | string )) }}{% endfor -%}{% endfor -%}{{ targets }}"
     vtctld_targets: "{% set targets = [] -%}{% for host in groups['vtctld'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':'+ (vtctld_port | default(15000) | string )) }}{% endfor -%}{{ targets }}"
     node_targets: "{% set targets= [] -%}{% for host in groups['all'] -%}{{ targets.append( hostvars[host]['ansible_default_ipv4']['address']+':9100' ) }}{% endfor -%}{{ targets }}"
-    prometheus_skip_install: true
+    prometheus_skip_install: false
     prometheus_version: 2.49.1
     prometheus_global:
       scrape_interval: 15s

ansible/roles/etcd/tasks/main.yml

@@ -10,6 +10,8 @@
 # limitations under the License.
 
 ---
+- include_tasks: clean.yml
+
 - include_tasks: install.yml
   when: not etcd_launch | bool
 

ansible/roles/macrobench/tasks/main.yml

@@ -14,6 +14,11 @@
   become: yes
   become_user: root
   block:
+    - name: Remove old arewefastyet
+      shell: |
+        rm -Rf /go/src/github.com/vitessio/arewefastyet
+      changed_when: false
+
     - name: git clone arewefastyet
       git:
         repo: "{{ arewefastyet_git_repo }}"

ansible/roles/sysbench/tasks/install.yml

@@ -10,36 +10,33 @@
 # limitations under the License.
 
 ---
-- name: Install Sysbench
-  when: install_sysbench is defined
-  block:
-    - name: Update Sysbench
-      become: yes
-      become_user: root
-      git:
-        repo: https://github.com/planetscale/sysbench
-        dest: /src/sysbench
-        version: master
-        depth: 1
-        force: 1
+- name: Update Sysbench
+  become: yes
+  become_user: root
+  git:
+    repo: https://github.com/planetscale/sysbench
+    dest: /src/sysbench
+    version: master
+    depth: 1
+    force: 1
 
-    - name: Build sysbench
-      become: yes
-      become_user: root
-      shell: |
-        cd /src/sysbench
-        ./autogen.sh
-        ./configure
-        make -j
-        make install
-      changed_when: false
+- name: Build sysbench
+  become: yes
+  become_user: root
+  shell: |
+    cd /src/sysbench
+    ./autogen.sh
+    ./configure
+    make -j
+    make install
+  changed_when: false
 
-    - name: Get tpcc
-      become: yes
-      become_user: root
-      git:
-        repo: https://github.com/planetscale/sysbench-tpcc
-        dest: /src/sysbench-tpcc
-        version: master
-        depth: 1
-        force: 1
+- name: Get tpcc
+  become: yes
+  become_user: root
+  git:
+    repo: https://github.com/planetscale/sysbench-tpcc
+    dest: /src/sysbench-tpcc
+    version: master
+    depth: 1
+    force: 1

ansible/roles/vitess_build/tasks/main.yml

@@ -10,19 +10,12 @@
 # limitations under the License.
 
 ---
-- name: ensure vitess tmp directory
-  become: yes
-  become_user: root
-  file:
-    state: directory
-    path: '{{ vitess_root }}/tmp'
-    owner: '{{ vitess_user }}'
-    group: '{{ vitess_group }}'
+- name: Stop existing Vitess to build a new one
+  include_tasks: clean.yml
 
 - name: Update Vitess
   become: yes
   become_user: root
-  when: commit_previous_exec != vitess_git_version or not clean_previous_exec | bool
   block:
     - name: Fetch Updated Vitess
       include_tasks: install_vitess.yml
@@ -62,7 +55,6 @@
       changed_when: false
 
 - name: Disbale AppArmor
-  when: not clean_previous_exec | bool
   block:
     - name: link apparmor
       file:

ansible/roles/vtctld/tasks/main.yml

@@ -10,12 +10,18 @@
 # limitations under the License.
 
 ---
+- name: Create users
+  include_role:
+    name: "vitess_build"
+    tasks_from: create_users
+
 - name: make sure vtctld is stopped
   when: clean_vtctld | bool
   include_role:
     name: "vtctld"
     tasks_from: "clean"
 
+
 - name: ensure vitess directories
   become: yes
   become_user: root

ansible/roles/vtgate/tasks/main.yml

@@ -16,6 +16,11 @@
     name: "vtgate"
     tasks_from: clean
 
+- name: Create users
+  include_role:
+    name: "vitess_build"
+    tasks_from: create_users
+
 - name: ensure vitess directories
   become: yes
   become_user: root

ansible/roles/vttablet/tasks/main.yml

@@ -16,6 +16,11 @@
     name: "vttablet"
     tasks_from: "clean"
 
+- name: Create users
+  include_role:
+    name: "vitess_build"
+    tasks_from: create_users
+
 - name: ensure vitess directories
   become: yes
   become_user: root
@@ -33,9 +38,8 @@
 - name: install systemd units
   include_tasks: systemd.yml
 
-- name: Install config files
+- name: install config files
   include_tasks: vttablet.yml
   loop: '{{ tablets }}'
   loop_control:
     loop_var: tablet
-

go/exec/exec.go

@@ -141,12 +141,6 @@ type Exec struct {
 	vitessConfig    vitessConfig
 
 	vitessSchemaPath string
-
-	// We always fetch the previous execution to let Ansible skip certain steps of the benchmark
-	// execution. Like cleaning up at the start of the execution, this is not required when the
-	// previous execution was clean and did not fail. Moreover, we can skip the fetch and build
-	// of Vitess if the previous execution share the same commit as the current execution.
-	previousExecution *Exec
 }
 
 const (
@@ -261,13 +255,6 @@ func (e *Exec) Prepare() error {
 		return err
 	}
 
-	// get previous benchmark
-	previousExec, err := getPreviousExecution(e.clientDB)
-	if err != nil {
-		return err
-	}
-	e.previousExecution = previousExec
-
 	// insert new exec in SQL
 	if _, err = e.clientDB.Insert(
 		"INSERT INTO execution(uuid, status, source, git_ref, type, pull_nb, go_version) VALUES(?, ?, ?, ?, ?, ?, ?)",
@@ -393,10 +380,6 @@ func (e *Exec) prepareAnsibleForExecution() error {
 	// runtime related values
 	e.AnsibleConfig.AddExtraVar(ansible.KeyGoVersion, e.GolangVersion)
 
-	// previous execution
-	e.AnsibleConfig.AddExtraVar(ansible.KeyCleanPreviousExec, e.previousExecution.Status == StatusFinished)
-	e.AnsibleConfig.AddExtraVar(ansible.KeyCommitPreviousExec, e.previousExecution.GitRef)
-
 	// stats database related values
 	e.statsRemoteDBConfig.AddToAnsible(&e.AnsibleConfig)
 	return nil
@@ -438,29 +421,6 @@ func (e *Exec) defineVersionNameOfVitess() error {
 	return nil
 }
 
-func getPreviousExecution(client storage.SQLClient) (*Exec, error) {
-	query := "SELECT uuid, status, git_ref, started_at, finished_at, source, type, pull_nb, go_version FROM execution ORDER BY started_at DESC LIMIT 1"
-	result, err := client.Select(query)
-	if err != nil {
-		return nil, err
-	}
-	defer result.Close()
-
-	var eUUID string
-	exec := &Exec{}
-	if result.Next() {
-		err = result.Scan(&eUUID, &exec.Status, &exec.GitRef, &exec.StartedAt, &exec.FinishedAt, &exec.Source, &exec.TypeOf, &exec.PullNB, &exec.GolangVersion)
-		if err != nil {
-			return nil, err
-		}
-		exec.UUID, err = uuid.Parse(eUUID)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return exec, nil
-}
-
 func GetRecentExecutions(client storage.SQLClient) ([]*Exec, error) {
 	var res []*Exec
 	query := "SELECT uuid, status, git_ref, started_at, finished_at, source, type, pull_nb, go_version FROM execution ORDER BY started_at DESC LIMIT 50"
@@ -564,6 +524,23 @@ func GetPreviousFromSourceMacrobenchmark(client storage.SQLClient, source, typeO
 	return
 }
 
+// GetLatestDailyJobForMicrobenchmarks will fetch and return the commit sha for which
+// the last daily job for microbenchmarks was run
+func GetLatestDailyJobForMicrobenchmarks(client storage.SQLClient) (gitSha string, err error) {
+	query := "select git_ref from execution where source = \"cron\" and status = \"finished\" and type = \"micro\" order by started_at desc limit 1"
+	rows, err := client.Select(query)
+	if err != nil {
+		return "", err
+	}
+
+	defer rows.Close()
+	for rows.Next() {
+		err = rows.Scan(&gitSha)
+		return gitSha, err
+	}
+	return "", nil
+}
+
 // GetLatestDailyJobForMacrobenchmarks will fetch and return the commit sha for which
 // the last daily job for macrobenchmarks was run
 func GetLatestDailyJobForMacrobenchmarks(client storage.SQLClient) (gitSha string, err error) {
@@ -601,11 +578,50 @@ func ExistsMacrobenchmark(client storage.SQLClient, gitRef, source, typeOf, stat
 	return result.Next(), nil
 }
 
+func ExistsMacrobenchmarkStartedToday(client storage.SQLClient, gitRef, source, typeOf, planner, status string) (bool, error) {
+	query := fmt.Sprintf("SELECT uuid FROM execution e, macrobenchmark m WHERE e.status = '%s' AND e.git_ref = ? AND e.type = ? AND e.source = ? AND m.vtgate_planner_version = ? AND e.uuid = m.exec_uuid AND e.started_at >= CURDATE()", status)
+	result, err := client.Select(query, gitRef, typeOf, source, planner)
+	if err != nil {
+		return false, err
+	}
+	exists := result.Next()
+	result.Close()
+	if exists {
+		return true, nil
+	}
+	query = fmt.Sprintf("SELECT uuid FROM execution e WHERE e.status = '%s' AND e.git_ref = ? AND e.type = ? AND e.source = ? AND e.started_at >= CURDATE()", status)
+	result, err = client.Select(query, gitRef, typeOf, source)
+	if err != nil {
+		return false, err
+	}
+	defer result.Close()
+	for result.Next() {
+		var exec_uuid string
+		err = result.Scan(&exec_uuid)
+		if err != nil {
+			return false, err
+		}
+		query = "SELECT exec_uuid FROM macrobenchmark m WHERE m.exec_uuid = ?"
+		resultMacro, err := client.Select(query, exec_uuid)
+		if err != nil {
+			return false, err
+		}
+		defer resultMacro.Close()
+
+		next := resultMacro.Next()
+		resultMacro.Close()
+		if !next {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func DeleteExecution(client storage.SQLClient, gitRef, UUID, source string) error {
 	query := fmt.Sprintf("DELETE FROM execution WHERE uuid LIKE '%%%s%%' AND git_ref LIKE '%%%s%%' AND source = '%s'", UUID, gitRef, source)
 	_, err := client.Select(query)
 	if err != nil {
 		return err
 	}
 	return nil
-}
+}

go/infra/ansible/vars.go

@@ -89,15 +89,6 @@ const (
 	// the execution of the benchmark.
 	KeyGoVersion = "golang_gover"
 
-	// Previous execution keys
-
-	// KeyCleanPreviousExec tells whether the previous execution was a clean benchmark or not
-	// this is useful when doing optimization to skip certain steps of the setup
-	KeyCleanPreviousExec = "clean_previous_exec"
-
-	// KeyCommitPreviousExec is the commit SHA of the previous execution.
-	KeyCommitPreviousExec = "commit_previous_exec"
-
 	// Stats database related keys
 
 	// KeyStatsDBHost corresponding value in the map is the hostname for the stats