-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[airflow] Avoid implicit DAG schedule (AIR301) (#14581)
- Loading branch information
Showing
9 changed files
with
133 additions
and
2 deletions.
There are no files selected for viewing
15 changes: 15 additions & 0 deletions
15
crates/ruff_linter/resources/test/fixtures/airflow/AIR301.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
from airflow import DAG, dag | ||
|
||
DAG(dag_id="class_default_schedule") | ||
|
||
DAG(dag_id="class_schedule", schedule="@hourly") | ||
|
||
|
||
@dag() | ||
def decorator_default_schedule(): | ||
pass | ||
|
||
|
||
@dag(schedule="0 * * * *") | ||
def decorator_schedule(): | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
84 changes: 84 additions & 0 deletions
84
crates/ruff_linter/src/rules/airflow/rules/dag_schedule_argument.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,84 @@ | ||
use ruff_diagnostics::{Diagnostic, Violation}; | ||
use ruff_macros::{derive_message_formats, violation}; | ||
use ruff_python_ast::Expr; | ||
use ruff_python_ast::{self as ast}; | ||
use ruff_python_semantic::Modules; | ||
use ruff_text_size::Ranged; | ||
|
||
use crate::checkers::ast::Checker; | ||
|
||
/// ## What it does | ||
/// Checks for a `DAG()` class or `@dag()` decorator without an explicit | ||
/// `schedule` parameter. | ||
/// | ||
/// ## Why is this bad? | ||
/// The default `schedule` value on Airflow 2 is `timedelta(days=1)`, which is | ||
/// almost never what a user is looking for. Airflow 3 changes this the default | ||
/// to *None*, and would break existing DAGs using the implicit default. | ||
/// | ||
/// If your DAG does not have an explicit `schedule` argument, Airflow 2 | ||
/// schedules a run for it every day (at the time determined by `start_date`). | ||
/// Such a DAG will no longer be scheduled on Airflow 3 at all, without any | ||
/// exceptions or other messages visible to the user. | ||
/// | ||
/// ## Example | ||
/// ```python | ||
/// from airflow import DAG | ||
/// | ||
/// | ||
/// # Using the implicit default schedule. | ||
/// dag = DAG(dag_id="my_dag") | ||
/// ``` | ||
/// | ||
/// Use instead: | ||
/// ```python | ||
/// from datetime import timedelta | ||
/// | ||
/// from airflow import DAG | ||
/// | ||
/// | ||
/// dag = DAG(dag_id="my_dag", schedule=timedelta(days=1)) | ||
/// ``` | ||
#[violation] | ||
pub struct AirflowDagNoScheduleArgument; | ||
|
||
impl Violation for AirflowDagNoScheduleArgument { | ||
#[derive_message_formats] | ||
fn message(&self) -> String { | ||
"DAG should have an explicit `schedule` argument".to_string() | ||
} | ||
} | ||
|
||
/// AIR301 | ||
pub(crate) fn dag_no_schedule_argument(checker: &mut Checker, expr: &Expr) { | ||
if !checker.semantic().seen_module(Modules::AIRFLOW) { | ||
return; | ||
} | ||
|
||
// Don't check non-call expressions. | ||
let Expr::Call(ast::ExprCall { | ||
func, arguments, .. | ||
}) = expr | ||
else { | ||
return; | ||
}; | ||
|
||
// We don't do anything unless this is a `DAG` (class) or `dag` (decorator | ||
// function) from Airflow. | ||
if !checker | ||
.semantic() | ||
.resolve_qualified_name(func) | ||
.is_some_and(|qualname| matches!(qualname.segments(), ["airflow", .., "DAG" | "dag"])) | ||
{ | ||
return; | ||
} | ||
|
||
// If there's a `schedule` keyword argument, we are good. | ||
if arguments.find_keyword("schedule").is_some() { | ||
return; | ||
} | ||
|
||
// Produce a diagnostic when the `schedule` keyword argument is not found. | ||
let diagnostic = Diagnostic::new(AirflowDagNoScheduleArgument, expr.range()); | ||
checker.diagnostics.push(diagnostic); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,5 @@ | ||
pub(crate) use dag_schedule_argument::*; | ||
pub(crate) use task_variable_name::*; | ||
|
||
mod dag_schedule_argument; | ||
mod task_variable_name; |
20 changes: 20 additions & 0 deletions
20
...ter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR301_AIR301.py.snap
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
--- | ||
source: crates/ruff_linter/src/rules/airflow/mod.rs | ||
--- | ||
AIR301.py:3:1: AIR301 DAG should have an explicit `schedule` argument | ||
| | ||
1 | from airflow import DAG, dag | ||
2 | | ||
3 | DAG(dag_id="class_default_schedule") | ||
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AIR301 | ||
4 | | ||
5 | DAG(dag_id="class_schedule", schedule="@hourly") | ||
| | ||
|
||
AIR301.py:8:2: AIR301 DAG should have an explicit `schedule` argument | ||
| | ||
8 | @dag() | ||
| ^^^^^ AIR301 | ||
9 | def decorator_default_schedule(): | ||
10 | pass | ||
| |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.