diff --git a/README.md b/README.md
index f163eb4..bd5ac54 100644
--- a/README.md
+++ b/README.md
@@ -136,18 +136,38 @@ Migrations is heavily inspired by the Oban approach. To add to the project table
 mix ecto.gen.migration add_kanta_translations_table
 ```
 
-Open the generated migration file and set up `up` and `down` functions:
+Open the generated migration file and set up `up` and `down` functions.
+
+### PostgreSQL
+
+```elixir
+defmodule MyApp.Repo.Migrations.AddKantaTranslationsTable do
+  use Ecto.Migration
+
+  def up do
+    Kanta.Migration.up(version: 4)
+  end
+
+  # We specify `version: 1` because we want to rollback all the way down including the first migration.
+  def down do
+    Kanta.Migration.down(version: 1)
+  end
+end
+```
+
+### SQLite3
 
 ```elixir
 defmodule MyApp.Repo.Migrations.AddKantaTranslationsTable do
   use Ecto.Migration
 
   def up do
-    Kanta.Migration.up(version: 3, prefix: prefix()) # Prefix is needed if you are using multitenancy with i.e. triplex
+    Kanta.Migration.up(version: 3)
   end
 
+  # We specify `version: 1` because we want to rollback all the way down including the first migration.
   def down do
-    Kanta.Migration.down(version: 3, prefix: prefix()) # Prefix is needed if you are using multitenancy with i.e. triplex
+    Kanta.Migration.down(version: 1)
   end
 end
 ```
diff --git a/lib/kanta/migration.ex b/lib/kanta/migration.ex
index 8c117d5..6bed45c 100644
--- a/lib/kanta/migration.ex
+++ b/lib/kanta/migration.ex
@@ -55,7 +55,7 @@ defmodule Kanta.Migration do
 
     def up, do: Kanta.Migration.up(version: 11)
 
-    def down, do: Kanta.Migration.down(version: 11)
+    def down, do: Kanta.Migration.down(version: 1)
   end
   ```
 
@@ -169,7 +169,7 @@ defmodule Kanta.Migration do
 
   Run migrations down to and including a specified version:
 
-      Kanta.Migration.down(version: 5)
+      Kanta.Migration.down(version: 1)
 
   Run migrations in an alternate prefix:
 
diff --git a/lib/kanta/migrations/postgresql.ex b/lib/kanta/migrations/postgresql.ex
index 44d5d59..7399757 100644
--- a/lib/kanta/migrations/postgresql.ex
+++ b/lib/kanta/migrations/postgresql.ex
@@ -6,7 +6,7 @@ defmodule Kanta.Migrations.Postgresql do
   use Ecto.Migration
 
   @initial_version 1
-  @current_version 3
+  @current_version 4
   @default_prefix "public"
 
   @doc false
diff --git a/lib/kanta/migrations/postgresql/V04.ex b/lib/kanta/migrations/postgresql/V04.ex
deleted file mode 100644
index 5de52a1..0000000
--- a/lib/kanta/migrations/postgresql/V04.ex
+++ /dev/null
@@ -1,23 +0,0 @@
-defmodule Kanta.Migrations.Postgresql.V04 do
-  @moduledoc """
-  Kanta V4 Migrations
-  """
-
-  use Ecto.Migration
-
-  def up do
-    Kanta.Migration.up(version: 3)
-
-    execute(
-      "INSERT INTO kanta_contexts (name,inserted_at,updated_at) VALUES('default', NOW(), NOW()) ON CONFLICT (name) DO NOTHING;"
-    )
-
-    execute("UPDATE kanta_messages SET context_id=1 WHERE context_id IS NULL;")
-  end
-
-  def down do
-    execute("UPDATE kanta_messages SET context_id=NULL WHERE context_id=1;")
-
-    execute("DELETE FROM kanta_contexts WHERE name='default';")
-  end
-end
diff --git a/lib/kanta/migrations/postgresql/v01.ex b/lib/kanta/migrations/postgresql/v01.ex
index bd70682..0ca3658 100644
--- a/lib/kanta/migrations/postgresql/v01.ex
+++ b/lib/kanta/migrations/postgresql/v01.ex
@@ -1,6 +1,6 @@
 defmodule Kanta.Migrations.Postgresql.V01 do
   @moduledoc """
-  Kanta V1 Migrations
+  Kanta PostgreSQL V1 Migrations
   """
 
   use Ecto.Migration
diff --git a/lib/kanta/migrations/postgresql/v02.ex b/lib/kanta/migrations/postgresql/v02.ex
index a545584..518922d 100644
--- a/lib/kanta/migrations/postgresql/v02.ex
+++ b/lib/kanta/migrations/postgresql/v02.ex
@@ -1,6 +1,6 @@
 defmodule Kanta.Migrations.Postgresql.V02 do
   @moduledoc """
-  Kanta V2 Migrations
+  Kanta PostgreSQL V2 Migrations
   """
 
   use Ecto.Migration
@@ -10,13 +10,11 @@ defmodule Kanta.Migrations.Postgresql.V02 do
   @kanta_plural_translations "kanta_plural_translations"
 
   def up(opts) do
-    Kanta.Migration.up(version: 1)
     up_fuzzy_search(opts)
   end
 
   def down(opts) do
     down_fuzzy_search(opts)
-    Kanta.Migration.down(version: 1)
   end
 
   def up_fuzzy_search(opts) do
diff --git a/lib/kanta/migrations/postgresql/v03.ex b/lib/kanta/migrations/postgresql/v03.ex
index 33ab2f5..79c4644 100644
--- a/lib/kanta/migrations/postgresql/v03.ex
+++ b/lib/kanta/migrations/postgresql/v03.ex
@@ -1,6 +1,6 @@
 defmodule Kanta.Migrations.Postgresql.V03 do
   @moduledoc """
-  Kanta V3 Migrations
+  Kanta PostgreSQL V3 Migrations
   """
 
   use Ecto.Migration
@@ -10,8 +10,6 @@ defmodule Kanta.Migrations.Postgresql.V03 do
   @kanta_messages "kanta_messages"
 
   def up(opts) do
-    Kanta.Migration.up(version: 2)
-
     [
       &up_application_sources/1,
       &up_kanta_messages/1
@@ -21,12 +19,10 @@ defmodule Kanta.Migrations.Postgresql.V03 do
 
   def down(opts) do
     [
-      &down_application_sources/1,
-      &down_kanta_messages/1
+      &down_kanta_messages/1,
+      &down_application_sources/1
     ]
     |> Enum.each(&apply(&1, [opts]))
-
-    Kanta.Migration.down(version: 2)
   end
 
   def up_application_sources(_opts) do
diff --git a/lib/kanta/migrations/postgresql/v04.ex b/lib/kanta/migrations/postgresql/v04.ex
new file mode 100644
index 0000000..c387bb8
--- /dev/null
+++ b/lib/kanta/migrations/postgresql/v04.ex
@@ -0,0 +1,30 @@
+defmodule Kanta.Migrations.Postgresql.V04 do
+  @moduledoc """
+  Kanta PostgreSQL V4 Migrations
+  """
+
+  use Ecto.Migration
+
+  @doc """
+  Ensure that the `default` context exists.
+  """
+  def up(_opts) do
+    # Insert the 'default' context if it doesn't exist
+    execute("""
+    INSERT INTO kanta_contexts (name, inserted_at, updated_at)
+    VALUES ('default', NOW(), NOW())
+    ON CONFLICT (name) DO NOTHING;
+    """)
+
+    flush()
+
+    # Update messages with null context_id to use the 'default' context's ID
+    execute("""
+    UPDATE kanta_messages
+    SET context_id = (SELECT id FROM kanta_contexts WHERE name = 'default')
+    WHERE context_id IS NULL;
+    """)
+  end
+
+  def down(_opts), do: nil
+end
diff --git a/lib/kanta/migrations/sqlite3.ex b/lib/kanta/migrations/sqlite3.ex
index 6848fd7..e3ec507 100644
--- a/lib/kanta/migrations/sqlite3.ex
+++ b/lib/kanta/migrations/sqlite3.ex
@@ -6,7 +6,7 @@ defmodule Kanta.Migrations.SQLite3 do
   use Ecto.Migration
 
   @initial_version 1
-  @current_version 2
+  @current_version 3
 
   @doc false
   def initial_version, do: @initial_version
diff --git a/lib/kanta/migrations/sqlite3/v01.ex b/lib/kanta/migrations/sqlite3/v01.ex
index 14f12ea..1416170 100644
--- a/lib/kanta/migrations/sqlite3/v01.ex
+++ b/lib/kanta/migrations/sqlite3/v01.ex
@@ -1,5 +1,7 @@
 defmodule Kanta.Migrations.SQLite3.V01 do
-  @moduledoc false
+  @moduledoc """
+  Kanta SQLite3 V1 Migrations
+  """
 
   use Ecto.Migration
 
diff --git a/lib/kanta/migrations/sqlite3/v02.ex b/lib/kanta/migrations/sqlite3/v02.ex
index 797c0c1..1b7f0fd 100644
--- a/lib/kanta/migrations/sqlite3/v02.ex
+++ b/lib/kanta/migrations/sqlite3/v02.ex
@@ -1,6 +1,6 @@
 defmodule Kanta.Migrations.SQLite3.V02 do
   @moduledoc """
-  Kanta V2 Migrations
+  Kanta SQLite3 V2 Migrations
   """
 
   use Ecto.Migration
diff --git a/lib/kanta/migrations/sqlite3/v03.ex b/lib/kanta/migrations/sqlite3/v03.ex
new file mode 100644
index 0000000..1afb28e
--- /dev/null
+++ b/lib/kanta/migrations/sqlite3/v03.ex
@@ -0,0 +1,35 @@
+defmodule Kanta.Migrations.SQLite3.V03 do
+  @moduledoc """
+  Kanta SQLite3 V3 Migrations
+  """
+
+  use Ecto.Migration
+
+  @doc """
+  Ensure that the `default` context exists.
+  """
+  def up(_opts) do
+    # Insert the 'default' context if it doesn't exist
+    execute("""
+    INSERT INTO kanta_contexts (name, inserted_at, updated_at)
+    VALUES ('default', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+    ON CONFLICT (name) DO NOTHING;
+    """)
+
+    flush()
+
+    # Update messages with null context_id to use the 'default' context's ID
+    execute("""
+    UPDATE kanta_messages
+    SET context_id = (
+      SELECT id
+      FROM kanta_contexts
+      WHERE name = 'default'
+      LIMIT 1
+    )
+    WHERE context_id IS NULL;
+    """)
+  end
+
+  def down(_opts), do: nil
+end