Skip to content

Commit

Permalink
Simplify ORM for performance
Browse files Browse the repository at this point in the history 
Drop issue version selection from the ORM and corresponding queries to
increase performance. Correspondingly drop "bugs" from the ORM and add
"issue_version". Rename "version" and "issue_version" fields to
"version_num" and "issue_version_num" in the ORM queries to avoid a
clash with "issue_version" the object type.
  • Loading branch information
@spbnick
spbnick committed Oct 24, 2024
1 parent dad90ae commit c21c6f8
Show file tree
Hide file tree
Showing 20 changed files with 403 additions and 933 deletions.
18 changes: 8 additions & 10 deletions kcidb/db/bigquery/v04_00.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,22 +583,19 @@ class Schema(AbstractSchema):
id="STRING",
),
),
bug=dict(
issue=dict(
statement='SELECT\n'
' "" AS url,\n'
' "" AS subject,\n'
' FALSE AS culprit_code,\n'
' FALSE AS culprit_tool,\n'
' FALSE AS culprit_harness\n'
' "" AS id,\n'
' "" AS origin\n'
'FROM UNNEST([])',
id_field_types=dict(
url="STRING",
id="STRING",
),
),
issue=dict(
issue_version=dict(
statement='SELECT\n'
' "" AS id,\n'
' 0 AS version,\n'
' 0 AS version_num,\n'
' "" AS origin,\n'
' "" AS report_url,\n'
' "" AS report_subject,\n'
Expand All @@ -612,14 +609,15 @@ class Schema(AbstractSchema):
'FROM UNNEST([])',
id_field_types=dict(
id="STRING",
version_num="INTEGER",
),
),
incident=dict(
statement='SELECT\n'
' "" AS id,\n'
' "" AS origin,\n'
' "" AS issue_id,\n'
' 0 AS issue_version,\n'
' 0 AS issue_version_num,\n'
' "" AS build_id,\n'
' "" AS test_id,\n'
' FALSE AS present,\n'
Expand Down
75 changes: 12 additions & 63 deletions kcidb/db/bigquery/v04_01.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,34 +149,19 @@ class Schema(PreviousSchema):
# Queries for each type of raw object-oriented data
OO_QUERIES = merge_dicts(
PreviousSchema.OO_QUERIES,
bug=merge_dicts(
PreviousSchema.OO_QUERIES["bug"],
statement="SELECT\n"
" report_url AS url,\n"
" ANY_VALUE(report_subject) AS subject,\n"
" MAX(culprit.code) AS culprit_code,\n"
" MAX(culprit.tool) AS culprit_tool,\n"
" MAX(culprit.harness) AS culprit_harness\n"
"FROM (\n"
" SELECT\n"
" report_url,\n"
" report_subject,\n"
" culprit,\n"
" comment,\n"
" ROW_NUMBER() OVER (\n"
" PARTITION BY id\n"
" ORDER BY version DESC\n"
" ) AS precedence\n"
" FROM issues\n"
")\n"
"WHERE precedence = 1\n"
"GROUP BY report_url",
),
issue=merge_dicts(
PreviousSchema.OO_QUERIES["issue"],
statement="SELECT\n"
" id,\n"
" version,\n"
" ANY_VALUE(origin) AS origin\n"
"FROM issues\n"
"GROUP BY id",
),
issue_version=merge_dicts(
PreviousSchema.OO_QUERIES["issue_version"],
statement="SELECT\n"
" id,\n"
" version AS version_num,\n"
" origin,\n"
" report_url,\n"
" report_subject,\n"
Expand All @@ -187,57 +172,21 @@ class Schema(PreviousSchema):
" test_status,\n"
" comment,\n"
" misc\n"
"FROM (\n"
" SELECT\n"
" id,\n"
" version,\n"
" origin,\n"
" report_url,\n"
" report_subject,\n"
" culprit,\n"
" build_valid,\n"
" test_status,\n"
" comment,\n"
" misc,\n"
" ROW_NUMBER() OVER (\n"
" PARTITION BY id\n"
" ORDER BY version DESC\n"
" ) AS precedence\n"
" FROM issues\n"
")\n"
"WHERE precedence = 1",
"FROM issues",
),
incident=merge_dicts(
PreviousSchema.OO_QUERIES["incident"],
statement="SELECT\n"
" id,\n"
" origin,\n"
" issue_id,\n"
" issue_version,\n"
" issue_version AS issue_version_num,\n"
" build_id,\n"
" test_id,\n"
" present,\n"
" comment,\n"
" misc\n"
"FROM (\n"
" SELECT\n"
" id,\n"
" origin,\n"
" issue_id,\n"
" issue_version,\n"
" build_id,\n"
" test_id,\n"
" present,\n"
" comment,\n"
" misc,\n"
" DENSE_RANK() OVER (\n"
" PARTITION BY\n"
" issue_id, build_id, test_id\n"
" ORDER BY issue_version DESC\n"
" ) AS precedence\n"
" FROM incidents\n"
")\n"
"WHERE precedence = 1",
"FROM incidents",
),
)

Expand Down
26 changes: 10 additions & 16 deletions kcidb/db/postgresql/v04_00.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,26 +426,20 @@ class Schema(AbstractSchema):
misc=JSONColumn(),
)),
),
bug=dict(
issue=dict(
statement="SELECT\n"
" NULL AS url,\n"
" NULL AS subject,\n"
" NULL AS culprit_code,\n"
" NULL AS culprit_tool,\n"
" NULL AS culprit_harness\n"
" NULL AS id,\n"
" NULL AS origin\n"
"WHERE FALSE",
schema=Table(dict(
url=TextColumn(),
subject=TextColumn(),
culprit_code=BoolColumn(),
culprit_tool=BoolColumn(),
culprit_harness=BoolColumn(),
id=TextColumn(),
origin=TextColumn(),
)),
),
issue=dict(
issue_version=dict(
statement="SELECT\n"
" NULL AS id,\n"
" NULL AS version,\n"
" NULL AS version_num,\n"
" NULL AS origin,\n"
" NULL AS report_url,\n"
" NULL AS report_subject,\n"
Expand All @@ -459,7 +453,7 @@ class Schema(AbstractSchema):
"WHERE FALSE",
schema=Table(dict(
id=TextColumn(),
version=IntegerColumn(),
version_num=IntegerColumn(),
origin=TextColumn(),
report_url=TextColumn(),
report_subject=TextColumn(),
Expand All @@ -477,7 +471,7 @@ class Schema(AbstractSchema):
" NULL AS id,\n"
" NULL AS origin,\n"
" NULL AS issue_id,\n"
" NULL AS issue_version,\n"
" NULL AS issue_version_num,\n"
" NULL AS build_id,\n"
" NULL AS test_id,\n"
" NULL AS present,\n"
Expand All @@ -488,7 +482,7 @@ class Schema(AbstractSchema):
id=TextColumn(),
origin=TextColumn(),
issue_id=TextColumn(),
issue_version=IntegerColumn(),
issue_version_num=IntegerColumn(),
build_id=TextColumn(),
test_id=TextColumn(),
present=BoolColumn(),
Expand Down
78 changes: 12 additions & 66 deletions kcidb/db/postgresql/v04_01.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,35 +66,19 @@ class Schema(PreviousSchema):
# Both should have columns in the same order.
OO_QUERIES = merge_dicts(
PreviousSchema.OO_QUERIES,
bug=merge_dicts(
PreviousSchema.OO_QUERIES["bug"],
statement="SELECT\n"
" report_url AS url,\n"
" FIRST(report_subject) AS subject,\n"
" BOOL_OR(culprit_code) AS culprit_code,\n"
" BOOL_OR(culprit_tool) AS culprit_tool,\n"
" BOOL_OR(culprit_harness) AS culprit_harness\n"
"FROM (\n"
" SELECT\n"
" report_url,\n"
" report_subject,\n"
" culprit_code,\n"
" culprit_tool,\n"
" culprit_harness,\n"
" ROW_NUMBER() OVER (\n"
" PARTITION BY id\n"
" ORDER BY version DESC\n"
" ) AS precedence\n"
" FROM issues\n"
") AS prioritized_issues\n"
"WHERE precedence = 1\n"
"GROUP BY report_url",
),
issue=merge_dicts(
PreviousSchema.OO_QUERIES["issue"],
statement="SELECT\n"
" id,\n"
" version,\n"
" FIRST(origin) AS origin\n"
"FROM issues\n"
"GROUP BY id",
),
issue_version=merge_dicts(
PreviousSchema.OO_QUERIES["issue_version"],
statement="SELECT\n"
" id,\n"
" version AS version_num,\n"
" origin,\n"
" report_url,\n"
" report_subject,\n"
Expand All @@ -105,59 +89,21 @@ class Schema(PreviousSchema):
" test_status,\n"
" comment,\n"
" misc\n"
"FROM (\n"
" SELECT\n"
" id,\n"
" version,\n"
" origin,\n"
" report_url,\n"
" report_subject,\n"
" culprit_code,\n"
" culprit_tool,\n"
" culprit_harness,\n"
" build_valid,\n"
" test_status,\n"
" comment,\n"
" misc,\n"
" ROW_NUMBER() OVER (\n"
" PARTITION BY id\n"
" ORDER BY version DESC\n"
" ) AS precedence\n"
" FROM issues\n"
") AS prioritized_issues\n"
"WHERE precedence = 1",
"FROM issues",
),
incident=merge_dicts(
PreviousSchema.OO_QUERIES["incident"],
statement="SELECT\n"
" id,\n"
" origin,\n"
" issue_id,\n"
" issue_version,\n"
" issue_version AS issue_version_num,\n"
" build_id,\n"
" test_id,\n"
" present,\n"
" comment,\n"
" misc\n"
"FROM (\n"
" SELECT\n"
" id,\n"
" origin,\n"
" issue_id,\n"
" issue_version,\n"
" build_id,\n"
" test_id,\n"
" present,\n"
" comment,\n"
" misc,\n"
" DENSE_RANK() OVER (\n"
" PARTITION BY\n"
" issue_id, build_id, test_id\n"
" ORDER BY issue_version DESC\n"
" ) AS precedence\n"
" FROM incidents\n"
") AS prioritized_incidents\n"
"WHERE precedence = 1",
"FROM incidents",
),
)

Expand Down
26 changes: 10 additions & 16 deletions kcidb/db/sqlite/v04_00.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,26 +388,20 @@ class Schema(AbstractSchema):
misc=JSONColumn(),
)),
),
bug=dict(
issue=dict(
statement="SELECT\n"
" NULL AS url,\n"
" NULL AS subject,\n"
" NULL AS culprit_code,\n"
" NULL AS culprit_tool,\n"
" NULL AS culprit_harness\n"
" NULL AS id,\n"
" NULL AS origin\n"
"WHERE 0",
schema=Table(dict(
url=TextColumn(),
subject=TextColumn(),
culprit_code=BoolColumn(),
culprit_tool=BoolColumn(),
culprit_harness=BoolColumn(),
id=TextColumn(),
origin=TextColumn()
)),
),
issue=dict(
issue_version=dict(
statement="SELECT\n"
" NULL AS id,\n"
" NULL AS version,\n"
" NULL AS version_num,\n"
" NULL AS origin,\n"
" NULL AS report_url,\n"
" NULL AS report_subject,\n"
Expand All @@ -421,7 +415,7 @@ class Schema(AbstractSchema):
"WHERE 0",
schema=Table(dict(
id=TextColumn(),
version=IntegerColumn(),
version_num=IntegerColumn(),
origin=TextColumn(),
report_url=TextColumn(),
report_subject=TextColumn(),
Expand All @@ -439,7 +433,7 @@ class Schema(AbstractSchema):
" NULL AS id,\n"
" NULL AS origin,\n"
" NULL AS issue_id,\n"
" NULL AS issue_version,\n"
" NULL AS issue_version_num,\n"
" NULL AS build_id,\n"
" NULL AS test_id,\n"
" NULL AS present,\n"
Expand All @@ -450,7 +444,7 @@ class Schema(AbstractSchema):
id=TextColumn(),
origin=TextColumn(),
issue_id=TextColumn(),
issue_version=IntegerColumn(),
issue_version_num=IntegerColumn(),
build_id=TextColumn(),
test_id=TextColumn(),
present=BoolColumn(),
Expand Down
Loading

0 comments on commit c21c6f8

Please sign in to comment.