update workflow script

sfu-db · Apr 22, 2024 · 9eba52f · 9eba52f
1 parent 1ab90ce
commit 9eba52f
Show file tree

Hide file tree

Showing 7 changed files with 158 additions and 90 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -137,7 +137,7 @@ jobs:
     container: ubuntu:20.04
     strategy:
       matrix:
-        python-version: [[39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     services:
       # Label used to access the service container
       postgres:
@@ -182,16 +182,16 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set python version
-        run: |
-          echo "/opt/python/cp${{ matrix.python-version[0] }}-cp${{ matrix.python-version[0] }}/bin" >> $GITHUB_PATH
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Install tools
         run: |
           apt-get update
-          apt-get install -y curl postgresql-client build-essential python3-dev python3-pip pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev python3 python3-pip libicu66 libkrb5-dev libclang-dev
+          apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev libicu66 libkrb5-dev libclang-dev
           pip3 install mssql-cli
           pip3 install cli-helpers==2.2.0
-          ln -s /usr/bin/python3 /usr/bin/python
         env:
           DEBIAN_FRONTEND: noninteractive
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,7 +12,7 @@ jobs:
     container: quay.io/pypa/manylinux_2_28_x86_64
     strategy:
       matrix:
-        python-version: [[38, "3.8"], [39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
+        python-version: [[39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
     steps:
       - uses: actions/checkout@v4
 
@@ -162,7 +162,7 @@ jobs:
     strategy:
       matrix:
         os: ["windows-latest", "macos-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         include:
           - os: "macos-latest"
             features: "--features integrated-auth-gssapi"
@@ -226,7 +226,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
 
@@ -290,7 +290,7 @@ jobs:
     needs: [win-and-mac, linux, apple-arm]
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os: [macos-latest, ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v4

diff --git a/connectorx/src/sources/bigquery/mod.rs b/connectorx/src/sources/bigquery/mod.rs
@@ -1136,7 +1136,8 @@ impl<'r, 'a> Produce<'r, Option<DateTime<Utc>>> for BigQuerySourceParser {
                     * 1e9) as i64;
                 let secs = timestamp_ns / 1000000000;
                 let nsecs = (timestamp_ns % 1000000000) as u32;
-                NaiveDateTime::from_timestamp_opt(secs, nsecs).map(|ndt| DateTime::<Utc>::from_naive_utc_and_offset(ndt, Utc))
+                NaiveDateTime::from_timestamp_opt(secs, nsecs)
+                    .map(|ndt| DateTime::<Utc>::from_naive_utc_and_offset(ndt, Utc))
             }
         }
     }

diff --git a/connectorx/src/sources/oracle/mod.rs b/connectorx/src/sources/oracle/mod.rs
@@ -38,10 +38,7 @@ impl Dialect for OracleDialect {
     }
 
     fn is_identifier_part(&self, ch: char) -> bool {
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
-            || ch.is_ascii_digit()
-            || ch == '_'
+        ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_'
     }
 }
 

diff --git a/connectorx/src/sources/postgres/mod.rs b/connectorx/src/sources/postgres/mod.rs
@@ -165,10 +165,7 @@ where
             .unzip();
 
         self.names = names;
-        self.schema = pg_types
-            .iter()
-            .map(PostgresTypeSystem::from)
-            .collect();
+        self.schema = pg_types.iter().map(PostgresTypeSystem::from).collect();
         self.pg_schema = self
             .schema
             .iter()
@@ -512,7 +509,7 @@ impl<'r, 'a> Produce<'r, Option<NaiveDateTime>> for PostgresBinarySourcePartitio
             Some(postgres::types::Timestamp::PosInfinity) => Some(NaiveDateTime::MAX),
             Some(postgres::types::Timestamp::NegInfinity) => Some(NaiveDateTime::MIN),
             Some(postgres::types::Timestamp::Value(t)) => t,
-            None => None
+            None => None,
         }
     }
 }
@@ -545,7 +542,7 @@ impl<'r, 'a> Produce<'r, Option<DateTime<Utc>>> for PostgresBinarySourcePartitio
             Some(postgres::types::Timestamp::PosInfinity) => Some(DateTime::<Utc>::MAX_UTC),
             Some(postgres::types::Timestamp::NegInfinity) => Some(DateTime::<Utc>::MIN_UTC),
             Some(postgres::types::Timestamp::Value(t)) => t,
-            None => None
+            None => None,
         }
     }
 }
@@ -578,7 +575,7 @@ impl<'r, 'a> Produce<'r, Option<NaiveDate>> for PostgresBinarySourcePartitionPar
             Some(postgres::types::Date::PosInfinity) => Some(NaiveDate::MAX),
             Some(postgres::types::Date::NegInfinity) => Some(NaiveDate::MIN),
             Some(postgres::types::Date::Value(t)) => t,
-            None => None
+            None => None,
         }
     }
 }
@@ -853,7 +850,6 @@ impl<'r, 'a> Produce<'r, Option<Vec<bool>>> for PostgresCSVSourceParser<'a> {
     }
 }
 
-
 impl<'r, 'a> Produce<'r, Decimal> for PostgresCSVSourceParser<'a> {
     type Error = PostgresSourceError;
 
@@ -863,14 +859,13 @@ impl<'r, 'a> Produce<'r, Decimal> for PostgresCSVSourceParser<'a> {
         match &self.rowbuf[ridx][cidx][..] {
             "Infinity" => Decimal::MAX,
             "-Infinity" => Decimal::MIN,
-            v => v.parse().map_err(|_| {
-                ConnectorXError::cannot_produce::<Decimal>(Some(v.into()))
-            })?
+            v => v
+                .parse()
+                .map_err(|_| ConnectorXError::cannot_produce::<Decimal>(Some(v.into())))?,
         }
     }
 }
 
-
 impl<'r, 'a> Produce<'r, Option<Decimal>> for PostgresCSVSourceParser<'a> {
     type Error = PostgresSourceError;
 
@@ -881,14 +876,14 @@ impl<'r, 'a> Produce<'r, Option<Decimal>> for PostgresCSVSourceParser<'a> {
             "" => None,
             "Infinity" => Some(Decimal::MAX),
             "-Infinity" => Some(Decimal::MIN),
-            v => Some(v.parse().map_err(|_| {
-                ConnectorXError::cannot_produce::<Decimal>(Some(v.into()))
-            })?),
+            v => Some(
+                v.parse()
+                    .map_err(|_| ConnectorXError::cannot_produce::<Decimal>(Some(v.into())))?,
+            ),
         }
     }
 }
 
-
 impl<'r, 'a> Produce<'r, DateTime<Utc>> for PostgresCSVSourceParser<'a> {
     type Error = PostgresSourceError;
 
@@ -899,9 +894,9 @@ impl<'r, 'a> Produce<'r, DateTime<Utc>> for PostgresCSVSourceParser<'a> {
             "infinity" => DateTime::<Utc>::MAX_UTC,
             "-infinity" => DateTime::<Utc>::MIN_UTC,
             // postgres csv return example: 1970-01-01 00:00:01+00
-            v => format!("{}:00", v).parse().map_err(|_| {
-                    ConnectorXError::cannot_produce::<DateTime<Utc>>(Some(v.into()))
-            })?
+            v => format!("{}:00", v)
+                .parse()
+                .map_err(|_| ConnectorXError::cannot_produce::<DateTime<Utc>>(Some(v.into())))?,
         }
     }
 }
@@ -935,9 +930,8 @@ impl<'r, 'a> Produce<'r, NaiveDate> for PostgresCSVSourceParser<'a> {
         match &self.rowbuf[ridx][cidx][..] {
             "infinity" => NaiveDate::MAX,
             "-infinity" => NaiveDate::MIN,
-            v => NaiveDate::parse_from_str(v, "%Y-%m-%d").map_err(|_| {
-                ConnectorXError::cannot_produce::<NaiveDate>(Some(v.into()))
-            })?
+            v => NaiveDate::parse_from_str(v, "%Y-%m-%d")
+                .map_err(|_| ConnectorXError::cannot_produce::<NaiveDate>(Some(v.into())))?,
         }
     }
 }
@@ -969,13 +963,8 @@ impl<'r, 'a> Produce<'r, NaiveDateTime> for PostgresCSVSourceParser<'a> {
         match &self.rowbuf[ridx][cidx] {
             "infinity" => NaiveDateTime::MAX,
             "-infinity" => NaiveDateTime::MIN,
-            v => NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S").map_err(
-                |_| {
-                    ConnectorXError::cannot_produce::<NaiveDateTime>(Some(
-                        v.into(),
-                    ))
-                },
-            )?
+            v => NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S")
+                .map_err(|_| ConnectorXError::cannot_produce::<NaiveDateTime>(Some(v.into())))?,
         }
     }
 }
@@ -1245,9 +1234,8 @@ impl<'r, 'a> Produce<'r, Option<DateTime<Utc>>> for PostgresRawSourceParser<'a>
             Some(postgres::types::Timestamp::PosInfinity) => Some(DateTime::<Utc>::MAX_UTC),
             Some(postgres::types::Timestamp::NegInfinity) => Some(DateTime::<Utc>::MIN_UTC),
             Some(postgres::types::Timestamp::Value(t)) => t,
-            None => None
+            None => None,
         }
-
     }
 }
 
@@ -1279,9 +1267,8 @@ impl<'r, 'a> Produce<'r, Option<NaiveDateTime>> for PostgresRawSourceParser<'a>
             Some(postgres::types::Timestamp::PosInfinity) => Some(NaiveDateTime::MAX),
             Some(postgres::types::Timestamp::NegInfinity) => Some(NaiveDateTime::MIN),
             Some(postgres::types::Timestamp::Value(t)) => t,
-            None => None
+            None => None,
         }
-
     }
 }
 
@@ -1315,7 +1302,6 @@ impl<'r, 'a> Produce<'r, Option<NaiveDate>> for PostgresRawSourceParser<'a> {
             Some(postgres::types::Date::Value(t)) => t,
             None => None,
         }
-
     }
 }
 
@@ -1483,9 +1469,7 @@ impl<'r> Produce<'r, Decimal> for PostgresSimpleSourceParser {
                 Some(s) => s
                     .parse()
                     .map_err(|_| ConnectorXError::cannot_produce::<Decimal>(Some(s.into())))?,
-                None => throw!(anyhow!(
-                    "Cannot parse NULL in NOT NULL column."
-                )),
+                None => throw!(anyhow!("Cannot parse NULL in NOT NULL column.")),
             },
             SimpleQueryMessage::CommandComplete(c) => {
                 panic!("get command: {}", c);
@@ -1787,9 +1771,10 @@ impl<'r> Produce<'r, NaiveDate> for PostgresSimpleSourceParser {
                 Some(s) => match s {
                     "infinity" => NaiveDate::MAX,
                     "-infinity" => NaiveDate::MIN,
-                    s => NaiveDate::parse_from_str(s, "%Y-%m-%d")
-                        .map_err(|_| ConnectorXError::cannot_produce::<NaiveDate>(Some(s.into())))?,
-                }
+                    s => NaiveDate::parse_from_str(s, "%Y-%m-%d").map_err(|_| {
+                        ConnectorXError::cannot_produce::<NaiveDate>(Some(s.into()))
+                    })?,
+                },
                 None => throw!(anyhow!("Cannot parse NULL in non-NULL column.")),
             },
             SimpleQueryMessage::CommandComplete(c) => {

diff --git a/connectorx/src/transports/trino_arrow.rs b/connectorx/src/transports/trino_arrow.rs
@@ -1,9 +1,7 @@
 //! Transport from Trino Source to Arrow Destination.
 
 use crate::{
-    destinations::arrow::{
-        typesystem::ArrowTypeSystem, ArrowDestination, ArrowDestinationError,
-    },
+    destinations::arrow::{typesystem::ArrowTypeSystem, ArrowDestination, ArrowDestinationError},
     impl_transport,
     sources::trino::{TrinoSource, TrinoSourceError, TrinoTypeSystem},
     typesystem::TypeConversion,