Skip to content

Commit c8602c0

Browse files
pkvachix5
pkvach
authored and
ix5
committed
db: Make 'text' field in 'comments' table NOT NULL and handling data migration
This update introduces a schema migration to version 4 for the database, focusing on enhancing the 'comments' table. This ensures that the 'text' field in the 'comments' table will always have a value, which improves data consistency and integrity. See: - isso-comments#979 - isso-comments#994
1 parent 6c6d5d2 commit c8602c0

4 files changed

Lines changed: 294 additions & 32 deletions

File tree

‎CHANGES.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ Bugfixes & Improvements
4545
- Fix newline character handling in data-isso-* i18n strings (`#992`_, pkvach)
4646
- Add link logging for management of new comments in Stdout (`#1016`_, pkvach)
4747
- Change logging to include datetime and loglevel (`#1023`_, ix5)
48+
- Make 'text' field in 'comments' table NOT NULL and handling data migration (`#1019`_, pkvach)
4849

4950
.. _#951: https://github.com/posativ/isso/pull/951
5051
.. _#967: https://github.com/posativ/isso/pull/967
@@ -56,6 +57,7 @@ Bugfixes & Improvements
5657
.. _#992: https://github.com/isso-comments/isso/pull/992
5758
.. _#1016: https://github.com/isso-comments/isso/pull/1016
5859
.. _#1023: https://github.com/isso-comments/isso/pull/1023
60+
.. _#1019: https://github.com/isso-comments/isso/pull/1019
5961

6062
0.13.1.dev0 (2023-02-05)
6163
------------------------

‎isso/db/__init__.py‎

Lines changed: 99 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class SQLite3:
2222
a trigger for automated orphan removal.
2323
"""
2424

25-
MAX_VERSION = 3
25+
MAX_VERSION = 5
2626

2727
def __init__(self, path, conf):
2828

@@ -68,7 +68,7 @@ def migrate(self, to):
6868
if self.version >= to:
6969
return
7070

71-
logger.info("migrate database from version %i to %i", self.version, to)
71+
logger.info("Migrating database from version %i to %i", self.version, to)
7272

7373
# re-initialize voters blob due a bug in the bloomfilter signature
7474
# which added older commenter's ip addresses to the current voters blob
@@ -78,20 +78,36 @@ def migrate(self, to):
7878
bf = memoryview(Bloomfilter(iterable=["127.0.0.0"]).array)
7979

8080
with sqlite3.connect(self.path) as con:
81-
con.execute('UPDATE comments SET voters=?', (bf, ))
82-
con.execute('PRAGMA user_version = 1')
83-
logger.info("%i rows changed", con.total_changes)
81+
con.execute("BEGIN TRANSACTION")
82+
try:
83+
con.execute('UPDATE comments SET voters=?', (bf, ))
84+
con.execute('PRAGMA user_version = 1')
85+
con.execute("COMMIT")
86+
logger.info("Migrating DB version 0 to 1 by re-initializing voters blob, %i rows changed",
87+
con.total_changes)
88+
except sqlite3.Error as e:
89+
con.execute("ROLLBACK")
90+
logger.error("Migrating DB version 0 to 1 failed: %s", e)
91+
raise RuntimeError("Migrating DB version 0 to 1 failed: %s" % e)
8492

8593
# move [general] session-key to database
8694
if self.version == 1:
8795

8896
with sqlite3.connect(self.path) as con:
89-
if self.conf.has_option("general", "session-key"):
90-
con.execute('UPDATE preferences SET value=? WHERE key=?', (
91-
self.conf.get("general", "session-key"), "session-key"))
92-
93-
con.execute('PRAGMA user_version = 2')
94-
logger.info("%i rows changed", con.total_changes)
97+
con.execute("BEGIN TRANSACTION")
98+
try:
99+
if self.conf.has_option("general", "session-key"):
100+
con.execute('UPDATE preferences SET value=? WHERE key=?', (
101+
self.conf.get("general", "session-key"), "session-key"))
102+
103+
con.execute('PRAGMA user_version = 2')
104+
con.execute("COMMIT")
105+
logger.info("Migrating DB version 1 to 2 by moving session-key to database, %i rows changed",
106+
con.total_changes)
107+
except sqlite3.Error as e:
108+
con.execute("ROLLBACK")
109+
logger.error("Migrating DB version 1 to 2 failed: %s", e)
110+
raise RuntimeError("Migrating DB version 1 to 2 failed: %s" % e)
95111

96112
# limit max. nesting level to 1
97113
if self.version == 2:
@@ -114,10 +130,76 @@ def first(rv):
114130
ids.extend(rv)
115131
flattened[id].update(set(rv))
116132

117-
for id in flattened.keys():
118-
for n in flattened[id]:
119-
con.execute(
120-
"UPDATE comments SET parent=? WHERE id=?", (id, n))
133+
con.execute("BEGIN TRANSACTION")
134+
try:
135+
for id in flattened.keys():
136+
for n in flattened[id]:
137+
con.execute(
138+
"UPDATE comments SET parent=? WHERE id=?", (id, n))
139+
140+
con.execute('PRAGMA user_version = 3')
141+
con.execute("COMMIT")
142+
logger.info("Migrating DB version 2 to 3 by limiting nesting level to 1, %i rows changed",
143+
con.total_changes)
144+
except sqlite3.Error as e:
145+
con.execute("ROLLBACK")
146+
logger.error("Migrating DB version 2 to 3 failed: %s", e)
147+
raise RuntimeError("Migrating DB version 2 to 3 failed: %s" % e)
148+
149+
# add notification field to comments (moved from Comments class to migration)
150+
if self.version == 3:
151+
with sqlite3.connect(self.path) as con:
152+
self.migrate_to_version_4(con)
153+
154+
# "text" field in "comments" became NOT NULL
155+
if self.version == 4:
156+
with sqlite3.connect(self.path) as con:
157+
con.execute("BEGIN TRANSACTION")
158+
con.execute("UPDATE comments SET text = '' WHERE text IS NULL")
159+
160+
# create new table with NOT NULL constraint for "text" field
161+
con.execute(Comments.create_table_query("comments_new"))
162+
163+
try:
164+
# copy data from old table to new table
165+
con.execute("""
166+
INSERT INTO comments_new (
167+
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
168+
)
169+
SELECT
170+
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
171+
FROM comments
172+
""")
173+
174+
# swap tables and drop old table
175+
con.execute("ALTER TABLE comments RENAME TO comments_backup_v4")
176+
con.execute("ALTER TABLE comments_new RENAME TO comments")
177+
con.execute("DROP TABLE comments_backup_v4")
178+
179+
con.execute('PRAGMA user_version = 5')
180+
con.execute("COMMIT")
181+
logger.info("Migrating DB version 4 to 5 by setting empty comments.text to '', %i rows changed",
182+
con.total_changes)
183+
except sqlite3.Error as e:
184+
con.execute("ROLLBACK")
185+
logger.error("Migrating DB version 4 to 5 failed: %s", e)
186+
raise RuntimeError("Migrating DB version 4 to 5 failed: %s" % e)
187+
188+
def migrate_to_version_4(self, con):
189+
# check if "notification" column exists in "comments" table
190+
rv = con.execute("PRAGMA table_info(comments)").fetchall()
191+
if any([row[1] == 'notification' for row in rv]):
192+
logger.info("Migrating DB version 3 to 4 skipped, 'notification' field already exists in comments")
193+
con.execute('PRAGMA user_version = 4')
194+
return
121195

122-
con.execute('PRAGMA user_version = 3')
123-
logger.info("%i rows changed", con.total_changes)
196+
con.execute("BEGIN TRANSACTION")
197+
try:
198+
con.execute('ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;')
199+
con.execute('PRAGMA user_version = 4')
200+
con.execute("COMMIT")
201+
logger.info("Migrating DB version 3 to 4 by adding 'notification' field to comments")
202+
except sqlite3.Error as e:
203+
con.execute("ROLLBACK")
204+
logger.error("Migrating DB version 3 to 4 failed: %s", e)
205+
raise RuntimeError("Migrating DB version 3 to 4 failed: %s" % e)

‎isso/db/comments.py‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,34 @@ class Comments:
3131
'remote_addr', 'text', 'author', 'email', 'website',
3232
'likes', 'dislikes', 'voters', 'notification']
3333

34+
# This method is used in the migration script from version 4 to 5.
35+
# You need to write a new migration if you change the database schema!
36+
@staticmethod
37+
def create_table_query(table_name):
38+
return f'''
39+
CREATE TABLE IF NOT EXISTS {table_name} (
40+
tid REFERENCES threads(id),
41+
id INTEGER PRIMARY KEY,
42+
parent INTEGER,
43+
created FLOAT NOT NULL,
44+
modified FLOAT,
45+
mode INTEGER,
46+
remote_addr VARCHAR,
47+
text VARCHAR NOT NULL,
48+
author VARCHAR,
49+
email VARCHAR,
50+
website VARCHAR,
51+
likes INTEGER DEFAULT 0,
52+
dislikes INTEGER DEFAULT 0,
53+
voters BLOB NOT NULL,
54+
notification INTEGER DEFAULT 0
55+
);
56+
'''
57+
3458
def __init__(self, db):
3559

3660
self.db = db
37-
self.db.execute([
38-
'CREATE TABLE IF NOT EXISTS comments (',
39-
' tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,',
40-
' created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,',
41-
' text VARCHAR NOT NULL, author VARCHAR, email VARCHAR, website VARCHAR,',
42-
' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,',
43-
' notification INTEGER DEFAULT 0);'])
44-
try:
45-
self.db.execute(['ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;'])
46-
except Exception:
47-
pass
61+
self.db.execute(Comments.create_table_query("comments"))
4862

4963
def add(self, uri, c):
5064
"""

0 commit comments

Comments
 (0)