From 1d61989e8514f4abf8f14baaf49a447e17841a68 Mon Sep 17 00:00:00 2001 From: Steve Williams <90905675+stevewgr@users.noreply.github.com> Date: Tue, 19 Nov 2024 02:10:27 -0500 Subject: [PATCH 1/4] Update database collation to MSSQL 2019 UTF-8. This change sets MSSQL 2019 as the minimum SQL Server version. For older versions, the collation can be reverted to `SQL_Latin1_General_CP1_CI_AS` by modifying the `import.ps1` script. However, UTF-8 compatibility is required for proper functionality. UTF-8 ensures consistent behavior regardless of the host system's settings. On Unix, UTF-8 is typically enabled by default. On Windows, it can be enabled under Region settings with "Beta: Use Unicode UTF-8 for worldwide language support." For details, see this discussion: https://github.com/ko4life-net/ko/pull/260#discussion_r1842825519 --- README.md | 4 ++-- import.ps1 | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index efc15f5..46435ed 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Brief explanation of the directory structure under `src`: ### Prerequisite - Being able to run powershell scripts. Note that if you're unable to run the scripts, it is because you need to allow powershell scripts to run on your system by setting the execution policy to bypass with the following powershell command: `Set-ExecutionPolicy Bypass -Scope CurrentUser` -- Microsoft SQL Server Express or Developer (confirmed to be working with versions 2022 and 2008) +- Microsoft SQL Server Express or Developer (confirmed to be working with versions 2022, minimum required 2019) - [SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) - [SQL Management Studio](https://learn.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms) - [Git](https://git-scm.com/download/win) (version 2.45.1 at the time of writing) @@ -45,7 +45,7 @@ During development we only create migration scripts to alter the current state o Every migration script will be prefixed with max 4 leading zeros. For example, `0001_insert_steve_user.sql` will contain an insert statement into `TB_USER` table. -Apart from the benifit of having the database under version control, this also makes it easy to use any SQL version you want. I use both 2008 and 2022 and it works perfectly fine with both. +Apart from the benifit of having the database under version control, this also makes it easy to use any SQL version you want. Release process should follow for every release we create in the [main ko project](https://github.com/ko4life-net/ko). The steps should look like this: - Main ko project needs a new release diff --git a/import.ps1 b/import.ps1 index b6964a5..f0a59ed 100644 --- a/import.ps1 +++ b/import.ps1 @@ -87,7 +87,12 @@ function RecreateDb { } MessageInfo "Creating database $db_name..." - InvokeSqlQuery -query "USE master; CREATE DATABASE [$db_name] COLLATE SQL_Latin1_General_CP1_CS_AS;" + # `Latin1_General_100_CS_AS_SC_UTF8` collation with UTF-8 support requires MSSQL 2019 or later. + # Details: https://learn.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-ver16#utf8 + # + # Archived migration scripts may fail from script 0021 due to the migration of `text` to `varchar`. + # If you want to test scripts with the initial setup, use `SQL_Latin1_General_CP1_CI_AS` collation instead. + InvokeSqlQuery -query "USE master; CREATE DATABASE [$db_name] COLLATE Latin1_General_100_CS_AS_SC_UTF8;" } function RunInitialDbScripts { From f72ae35eadd6f979308d62b04dbcccc945646623 Mon Sep 17 00:00:00 2001 From: Steve Williams <90905675+stevewgr@users.noreply.github.com> Date: Tue, 19 Nov 2024 02:24:20 -0500 Subject: [PATCH 2/4] Adjust VERSION table to match LoginServer's changes. Note that we decided to set strFile as varchar instead of nvarchar because of the newly updated UTF-8 compatible collation. --- src/migration/0024_alter_version_columns.sql | 15 +++++++++++ .../0024_alter_version_columns.sql.diff | 25 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/migration/0024_alter_version_columns.sql create mode 100644 src/migration/0024_alter_version_columns.sql.diff diff --git a/src/migration/0024_alter_version_columns.sql b/src/migration/0024_alter_version_columns.sql new file mode 100644 index 0000000..7e9e20b --- /dev/null +++ b/src/migration/0024_alter_version_columns.sql @@ -0,0 +1,15 @@ +-- Restructure VERSION table to match accomodate changes with the LogicServer's Version Manager. + +DROP TABLE [dbo].[VERSION]; +GO + +CREATE TABLE [dbo].[VERSION]( + [sVersion] [smallint] NOT NULL, + [strFile] [varchar](500) NULL, + [strPatchFileName] [varchar](13) NULL, + [sHistoryVersion] [smallint] NULL +) ON [PRIMARY] +GO + +INSERT [dbo].[VERSION] ([sVersion], [strFile], [strPatchFileName], [sHistoryVersion]) VALUES (1264, 'KnightOnLine.exe', 'patch1264.zip', 0) +GO diff --git a/src/migration/0024_alter_version_columns.sql.diff b/src/migration/0024_alter_version_columns.sql.diff new file mode 100644 index 0000000..96e2999 --- /dev/null +++ b/src/migration/0024_alter_version_columns.sql.diff @@ -0,0 +1,25 @@ +diff --git a/src/data/VERSION.sql b/src/data/VERSION.sql +index 2e88f9f..afef6fc 100644 +--- a/src/data/VERSION.sql ++++ b/src/data/VERSION.sql +@@ -1 +1 @@ +-INSERT [dbo].[VERSION] ([sVersion], [strFileName], [strCompressName], [sHistoryVersion]) VALUES (1089, N'eeee', N'eeee', 1089) ++INSERT [dbo].[VERSION] ([sVersion], [strFile], [strPatchFileName], [sHistoryVersion]) VALUES (1264, N'KnightOnLine.exe', N'patch1264.zip', 0) +diff --git a/src/schema/VERSION.sql b/src/schema/VERSION.sql +index 1f44939..6dac075 100644 +--- a/src/schema/VERSION.sql ++++ b/src/schema/VERSION.sql +@@ -3,9 +3,9 @@ GO + SET QUOTED_IDENTIFIER ON + GO + CREATE TABLE [dbo].[VERSION]( +- [sVersion] [int] NOT NULL, +- [strFileName] [varchar](40) NULL, +- [strCompressName] [varchar](10) NULL, +- [sHistoryVersion] [int] NULL ++ [sVersion] [smallint] NOT NULL, ++ [strFile] [varchar](500) NULL, ++ [strPatchFileName] [varchar](13) NULL, ++ [sHistoryVersion] [smallint] NULL + ) ON [PRIMARY] + GO From a72d7542228f11e5318b6374106fd4e2fdbed502 Mon Sep 17 00:00:00 2001 From: Steve Williams <90905675+stevewgr@users.noreply.github.com> Date: Tue, 19 Nov 2024 02:35:35 -0500 Subject: [PATCH 3/4] Update README.md to reflect the project's current state and setup. --- README.md | 87 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 46435ed..144374a 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,67 @@ # Knight Online Database -This repository contains scripts to build the database from scratch and set all the configurations needed for you to be able running the game. +This repository contains scripts to build the database from scratch and configure everything needed to run the game. -Brief explanation of the directory structure under `src`: -- data - Insert queries based on the current db state -- migration - [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) scripts to update the existing db state -- misc - Utility and useful scripts -- procedure - Stored procedures based on the current db state -- schema - Tables structure and constraints based on the current db state +### Directory Structure under `src`: +- **data**: Insert queries based on the current database state. +- **migration**: [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) scripts to update the existing database state. + - **archive**: [ko-db-archive](https://github.com/ko4life-net/ko-db-archive) archived migration scripts that has been released. +- **misc**: Utility and helper scripts. +- **procedure**: Stored procedures based on the current database state. +- **schema**: Table structures and constraints based on the current database state. -### Prerequisite -- Being able to run powershell scripts. Note that if you're unable to run the scripts, it is because you need to allow powershell scripts to run on your system by setting the execution policy to bypass with the following powershell command: `Set-ExecutionPolicy Bypass -Scope CurrentUser` -- Microsoft SQL Server Express or Developer (confirmed to be working with versions 2022, minimum required 2019) - - [SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) - - [SQL Management Studio](https://learn.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms) -- [Git](https://git-scm.com/download/win) (version 2.45.1 at the time of writing) -- [Python](https://www.python.org/downloads/) (version 3.12.3 at the time of writing) - - During installation in the `Advanced Options` make sure to tick `Add Python to environment variables` - - Once finished, install the required packages with the following command: `pip install -r requirements.txt` +### Prerequisites +- **PowerShell**: Enable script execution by running: `Set-ExecutionPolicy Bypass -Scope CurrentUser` +- **Git**: Install [Git](https://git-scm.com/download/win) (e.g., version 2.45.1). +- **Microsoft SQL Server**: + - Install SQL Server Express/Developer Edition (minimum: MSSQL 2019 for UTF-8 collation). + - [Download SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) + - Install [SQL Server Management Studio (SSMS)](https://learn.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms). +- **Python**: + - Install [Python](https://www.python.org/downloads/) (e.g., version 3.12.3). + - During setup, check **"Add Python to environment variables"** in `Advanced Options`. + - Install required packages: `pip install -r requirements.txt` -### Getting Started -Before running the `import.ps1` script, check that the `$server_name` variable matches with your current server name. If you installed SQL Server using the default instance, then the default arguments should be working fine, otherwise you can provide the script an argument with your custom server name. You can run the following command in powershell to know the names of your current SQL Servers: -```powershell -(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server').InstalledInstances -``` +### Getting Started -Assuming you set a custom name to your SQL Server instance, you may invoke the import command as follows: -```powershell -.\import.ps1 -server_name ".\MyCustomInstanceName" -``` +After completing the prerequisites, you can import the database into your SQL Server by running the import script: `.\import.ps1` -Once the import script finished importing the db, it will also invoke the `odbcad.ps1` script automatically for you to set odbc configurations so that the server files can connect with your db. +The script will automatically execute `odbcad.ps1` to configure ODBC settings, ensuring the server files can connect to your database. Once done, your database will be ready for use, and no additional configuration is needed. +Note: The main KO project clones this repository as a shallow submodule (without history). To browse the full history or inspect the migration scripts (a submodule of this project), unshallow the repository by running the following command in the root directory: `git fetch --unshallow` ### Development -Note that the development process is inspired from [different db development environments](https://docs.djangoproject.com/en/4.0/topics/migrations/), where a table is treated as a model. +The development process is inspired by [various database development environments](https://docs.djangoproject.com/en/4.0/topics/migrations/), where tables are treated as models. + +During development, create migration scripts to alter the current state of the database. The "base" refers to the generated scripts in the `data`, `procedure`, and `schema` directories. + +Migration scripts should be prefixed with a maximum of four leading zeros. For example: +- `0001_insert_steve_user.sql`: Contains an `INSERT` statement for the `TB_USER` table. + +Benefits of this approach: +- The database is version-controlled. +- You can easily use almost any SQL version you prefer. + + +### Release Process -During development we only create migration scripts to alter the current state of the base. The base here refers to the generated scripts in data, procedure and schema. +The release process for the [main KO project](https://github.com/ko4life-net/ko) should align with changes in the KO-DB project. Follow these steps: -Every migration script will be prefixed with max 4 leading zeros. For example, `0001_insert_steve_user.sql` will contain an insert statement into `TB_USER` table. +1. **Identify the Need for a KO-DB Release**: + - If the main KO project introduces changes that depend on KO-DB updates, ensure those updates are implemented in KO-DB first. + - Follow the steps in the [migration directory README.md](/src/migration/README.md) to prepare the changes. -Apart from the benifit of having the database under version control, this also makes it easy to use any SQL version you want. +2. **Create a KO-DB Release**: + - Once the database changes are ready, create a new KO-DB release with a version tag. -Release process should follow for every release we create in the [main ko project](https://github.com/ko4life-net/ko). The steps should look like this: -- Main ko project needs a new release - - Are there any pending migration scrips in the ko-db project? - - If yes, create a new ko-db release, which the steps are documented in the [migration dir README.md](/src/migration/README.md) - - If no, no need to update the ko-db project -- Main ko project sets the tag in the config script to the latest released ko-db version -- Main ko project creates a new release and after merging, changing the config script back to the ko-db master branch - - This is because the migration scripts in the ko-db project has been archived at this point, meaning newly added migration scripts will automatically affect the main ko project +3. **Update the Submodule**: + - In the main KO project, update the KO-DB submodule to reference the new release tag. -In other words, we use the `master` branch as the development and release branch, but when tagging, they are all fixed to a specific tag. -Maybe in the future this will change if it makes things difficult and we maintain a separate development branch. +**Best Practice**: +- If the KO-DB project has changes (e.g., cleaning inconsistencies or extending tables) that do not immediately impact the main KO project, you don’t need to draft a KO-DB release right away. +- However, if the main KO project is preparing a release and KO-DB has untagged updates, draft a new KO-DB release first. This ensures the main KO project can reference the latest KO-DB release tag. From 6655292120310297e0884c028518dff107e2e3f3 Mon Sep 17 00:00:00 2001 From: Steve Williams <90905675+stevewgr@users.noreply.github.com> Date: Tue, 19 Nov 2024 02:46:49 -0500 Subject: [PATCH 4/4] Disable automatic formatting for migration scripts. Formatting migration scripts is now optional. While formatting is helpful for table creation or altering procedures, it can be cumbersome for large insert queries. Therefore, automatic formatting has been disabled. --- format.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/format.ps1 b/format.ps1 index ce4f6da..3fddde3 100644 --- a/format.ps1 +++ b/format.ps1 @@ -5,7 +5,6 @@ param ( ) $dirs_to_format = @( - "$PSScriptRoot\src\migration\", "$PSScriptRoot\src\misc\", "$PSScriptRoot\src\procedure\" )