From 063102ae505afba38cf313dd4c7d0005a9066577 Mon Sep 17 00:00:00 2001 From: Rebecca Breu Date: Fri, 26 Mar 2021 19:55:08 +0100 Subject: [PATCH] Switch native file format to sqlite --- beeref/bee_json.py | 63 --------------- beeref/fileio/__init__.py | 35 +++++++++ beeref/fileio/sql.py | 147 +++++++++++++++++++++++++++++++++++ beeref/items.py | 46 +++-------- beeref/scene.py | 11 +-- beeref/view.py | 26 ++----- tests/fileio/__init__.py | 0 tests/fileio/test_sql.py | 156 ++++++++++++++++++++++++++++++++++++++ tests/test_bee_json.py | 30 -------- tests/test_items.py | 61 --------------- 10 files changed, 360 insertions(+), 215 deletions(-) delete mode 100644 beeref/bee_json.py create mode 100644 beeref/fileio/__init__.py create mode 100644 beeref/fileio/sql.py create mode 100644 tests/fileio/__init__.py create mode 100644 tests/fileio/test_sql.py delete mode 100644 tests/test_bee_json.py diff --git a/beeref/bee_json.py b/beeref/bee_json.py deleted file mode 100644 index 233339e..0000000 --- a/beeref/bee_json.py +++ /dev/null @@ -1,63 +0,0 @@ -# This file is part of BeeRef. -# -# BeeRef is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# BeeRef is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with BeeRef. If not, see . - -"""BeeRef's native file handling. - -BeeRef files are JSON files with images embedded as base64-encoded PNG data. -""" - -import json - -from beeref.items import BeePixmapItem - - -class BeeJSONEncoder(json.JSONEncoder): - """If an object defines the method ``to_bee_json``, use it to - serialize the object to JSON.""" - - def default(self, obj): - if hasattr(obj, 'to_bee_json'): - return obj.to_bee_json() - return json.JSONEncoder.default(self, obj) - - -def dumps(obj): - return json.dumps(obj, cls=BeeJSONEncoder) - - -class BeeJSONDecoder(json.JSONDecoder): - """If a dictionary in the JSON file defines the key ``cls``, we use - that class name and run the classmethod ``from_bee_json`` on it to - deserialize the object.""" - - bee_classes = [BeePixmapItem] - - def __init__(self, *args, **kwargs): - super().__init__(object_hook=self.object_hook, *args, **kwargs) - - def get_bee_class(self, cls): - for bee_class in self.bee_classes: - if bee_class.__name__ == cls: - return bee_class - - def object_hook(self, obj): - if 'cls' in obj: - bee_class = self.get_bee_class(obj['cls']) - return bee_class.from_bee_json(obj) - return obj - - -def loads(obj): - return json.loads(obj, cls=BeeJSONDecoder) diff --git a/beeref/fileio/__init__.py b/beeref/fileio/__init__.py new file mode 100644 index 0000000..abdb9fa --- /dev/null +++ b/beeref/fileio/__init__.py @@ -0,0 +1,35 @@ +# This file is part of BeeRef. +# +# BeeRef is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BeeRef is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BeeRef. If not, see . + +import logging + +from beeref.fileio.sql import SQLiteIO + + +__all__ = ['load', 'save'] +logger = logging.getLogger('BeeRef') + + +def load(filename, scene): + logger.info(f'Loading from file {filename}...') + io = SQLiteIO(filename, scene) + return io.read() + + +def save(filename, scene, create_new=False): + logger.info(f'Saving to file {filename}...') + io = SQLiteIO(filename, scene, create_new) + io.write() + logger.debug('Saved!') diff --git a/beeref/fileio/sql.py b/beeref/fileio/sql.py new file mode 100644 index 0000000..31cdcae --- /dev/null +++ b/beeref/fileio/sql.py @@ -0,0 +1,147 @@ +# This file is part of BeeRef. +# +# BeeRef is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BeeRef is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BeeRef. If not, see . + +"""BeeRef's native file format is using SQLite. For more info, see: + +https://www.sqlite.org/appfileformat.html +""" + +import os +import sqlite3 + +from PyQt6 import QtGui + +from beeref.items import BeePixmapItem + + +SCHEMA = [ + """ + CREATE TABLE items ( + id INTEGER PRIMARY KEY, + type TEXT NOT NULL, + pos_x REAL DEFAULT 0, + pos_y REAL DEFAULT 0, + scale REAL DEFAULT 1, + rotation REAL DEFAULT 0, + flip_h INTEGER DEFAULT 0, + flip_v INTEGER DEFAULT 0, + filename TEXT + ) + """, + """ + CREATE TABLE imgdata ( + id INTEGER PRIMARY KEY, + item_id INTEGER NOT NULL, + data BLOB, + FOREIGN KEY (item_id) + REFERENCES items (id) + ON DELETE CASCADE + ON UPDATE NO ACTION + ) + """, +] + + +class SQLiteIO: + USER_VERSION = 1 + APPLICATION_ID = 2060242126 + + def __init__(self, filename, scene, create_new=False): + self.scene = scene + self.filename = filename + self.create_new = create_new + # TBD: error handling (different for existing files/new files...) + if self.create_new and os.path.exists(self.filename): + os.remove(self.filename) + self.connection = sqlite3.connect(self.filename) + self.cursor = self.connection.cursor() + + def ex(self, *args, **kwargs): + return self.cursor.execute(*args, **kwargs) + + def exmany(self, *args, **kwargs): + return self.cursor.executemany(*args, **kwargs) + + def fetchone(self, *args, **kwargs): + self.ex(*args, **kwargs) + return self.cursor.fetchone() + + def fetchall(self, *args, **kwargs): + self.ex(*args, **kwargs) + return self.cursor.fetchall() + + def write_meta(self): + self.ex('PRAGMA application_id=%s' % self.APPLICATION_ID) + self.ex('PRAGMA user_version=%s' % self.USER_VERSION) + self.ex('PRAGMA foreign_keys=1') + + def create_schema_on_new(self): + if self.create_new: + for schema in SCHEMA: + self.ex(schema) + + def read(self): + rows = self.fetchall( + 'SELECT pos_x, pos_y, scale, filename, imgdata.data ' + 'FROM items ' + 'INNER JOIN imgdata on imgdata.item_id = items.id') + for row in rows: + item = BeePixmapItem(QtGui.QImage(), filename=row[3]) + item.pixmap_from_bytes(row[4]) + item.setPos(row[0], row[1]) + item.setScale(row[2]) + self.scene.addItem(item) + + def write(self): + self.write_meta() + self.create_schema_on_new() + + to_delete = self.fetchall('SELECT id from ITEMS') + for item in self.scene.items_for_save(): + if item.save_id and not self.create_new: + self.update_item(item) + to_delete.remove((item.save_id,)) + else: + self.insert_item(item) + self.delete_items(to_delete) + self.connection.commit() + + def delete_items(self, to_delete): + self.exmany('DELETE FROM items WHERE id=?', to_delete) + self.connection.commit() + + def insert_item(self, item): + self.ex( + 'INSERT INTO items (type, pos_x, pos_y, scale, filename) ' + 'VALUES (?, ?, ?, ?, ?) ', + ('pixmap', item.pos().x(), item.pos().y(), item.scale_factor, + item.filename)) + item.save_id = self.cursor.lastrowid + self.ex('INSERT INTO imgdata (item_id, data) VALUES (?, ?)', + (item.save_id, item.pixmap_to_bytes())) + self.connection.commit() + + def update_item(self, item): + """Update item data. + + We only update the item data, not the pixmap data, as pixmap + data never changes and is also time-consuming to save. + """ + self.ex( + 'UPDATE items SET pos_x=?, pos_y=?, scale=?, filename=? ' + 'WHERE id=?', + (item.pos().x(), item.pos().y(), item.scale_factor, + item.filename, item.save_id)) + self.connection.commit() diff --git a/beeref/items.py b/beeref/items.py index 91ed85b..0f5e3e9 100644 --- a/beeref/items.py +++ b/beeref/items.py @@ -17,7 +17,6 @@ text). """ -import base64 import logging from PyQt6 import QtCore, QtGui, QtWidgets @@ -36,8 +35,10 @@ class BeePixmapItem(QtWidgets.QGraphicsPixmapItem): logger.debug(f'Initialized image "{filename}" with dimensions: ' f'{self.width} x {self.height} at index {self.zValue()}') + self.save_id = None self.filename = filename self.scale_factor = 1 + self.setFlags( QtWidgets.QGraphicsItem.GraphicsItemFlags.ItemIsMovable | QtWidgets.QGraphicsItem.GraphicsItemFlags.ItemIsSelectable) @@ -63,47 +64,20 @@ class BeePixmapItem(QtWidgets.QGraphicsPixmapItem): def height(self): return self.pixmap().size().height() - def pixmap_to_str(self): - """Convert the pixmap data to a base64-encoded PNG for saving.""" + def pixmap_to_bytes(self): + """Convert the pixmap data to PNG bytestring.""" barray = QtCore.QByteArray() buffer = QtCore.QBuffer(barray) buffer.open(QtCore.QIODevice.OpenMode.WriteOnly) img = self.pixmap().toImage() img.save(buffer, 'PNG') - data = base64.b64encode(barray.data()) - return data.decode('ascii') + return barray.data() - @classmethod - def qimage_from_str(self, data): - """Read the image date from a base64-encoded PNG for loading.""" - img = QtGui.QImage() - img.loadFromData(base64.b64decode(data)) - return img - - def to_bee_json(self): - """For saving the item to BeeRefs native file format.""" - return { - 'cls': self.__class__.__name__, - 'scale': self.scale_factor, - 'pixmap': self.pixmap_to_str(), - 'pos': [self.pos().x(), self.pos().y()], - 'z': self.zValue(), - 'filename': self.filename, - } - - @classmethod - def from_bee_json(cls, obj): - """For loading an item from BeeRefs native file format.""" - img = cls.qimage_from_str(obj['pixmap']) - item = cls(img, filename=obj.get('filename')) - if 'scale' in obj: - item.setScale(obj['scale']) - if 'pos' in obj: - item.setPos(*obj['pos']) - if 'z' in obj: - item.setZValue(obj['z']) - - return item + def pixmap_from_bytes(self, data): + """Set image pimap from a bytestring.""" + pixmap = QtGui.QPixmap() + pixmap.loadFromData(data) + self.setPixmap(pixmap) def itemChange(self, change, value): if change == self.GraphicsItemChange.ItemSelectedChange: diff --git a/beeref/scene.py b/beeref/scene.py index 6f28d7a..6c6b243 100644 --- a/beeref/scene.py +++ b/beeref/scene.py @@ -112,13 +112,10 @@ class BeeGraphicsScene(QtWidgets.QGraphicsScene): self.move_active = False super().mouseReleaseEvent(event) - def items_for_export(self): - """Returns the items that are to be exported. + def items_for_save(self): + """Returns the items that are to be saved. - Items to be exported are items that implement ``to_bee_json``. + Items to be saved are items that have an save_id attribute. """ - # self.items() holds items in reverse order of addition, so we - # need to reverse it for export - return list(filter(lambda i: hasattr(i, 'to_bee_json'), - reversed(self.items()))) + return filter(lambda i: hasattr(i, 'save_id'), self.items()) diff --git a/beeref/view.py b/beeref/view.py index ec1d5bc..e0283a4 100644 --- a/beeref/view.py +++ b/beeref/view.py @@ -18,8 +18,8 @@ import logging from PyQt6 import QtCore, QtGui, QtWidgets from PyQt6.QtCore import Qt -from beeref import bee_json from beeref import commands +from beeref import fileio from beeref.gui import WelcomeOverlay from beeref.items import BeePixmapItem from beeref.scene import BeeGraphicsScene @@ -245,23 +245,12 @@ class BeeGraphicsView(QtWidgets.QGraphicsView): items.append(item) self.undo_stack.push(commands.InsertItems(self.scene, items)) - def save_to_file(self, filename): - logger.info(f'Saving to file {filename}') - dump = bee_json.dumps({'items': self.scene.items_for_export()}) - with open(filename, 'w') as f: - f.write(dump) - self.filename = filename - logger.debug('Saved!') - def open_from_file(self, filename): logger.info(f'Opening file {filename}') - with open(filename, 'r') as f: - items = bee_json.loads(f.read())['items'] - self.scene.clear() - self.undo_stack.clear() - for item in items: - self.scene.addItem(item) - self.filename = filename + self.scene.clear() + self.undo_stack.clear() + fileio.load(filename, self.scene) + self.filename = filename def on_action_open(self): filename, f = QtWidgets.QFileDialog.getOpenFileName( @@ -279,13 +268,14 @@ class BeeGraphicsView(QtWidgets.QGraphicsView): if filename: if not filename.endswith('.bee'): filename = f'{filename}.bee' - self.save_to_file(filename) + fileio.save(filename, self.scene, create_new=True) + self.filename = filename def on_action_save(self): if not self.filename: self.on_action_save_as() else: - self.save_to_file(self.filename) + fileio.save(self.filename, self.scene, create_new=False) def on_action_quit(self): logger.info('User quit. Exiting...') diff --git a/tests/fileio/__init__.py b/tests/fileio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fileio/test_sql.py b/tests/fileio/test_sql.py new file mode 100644 index 0000000..ed915e5 --- /dev/null +++ b/tests/fileio/test_sql.py @@ -0,0 +1,156 @@ +import os.path +from unittest.mock import MagicMock, patch + +from PyQt6 import QtGui + +from beeref.fileio.sql import SQLiteIO +from beeref.items import BeePixmapItem +from beeref.scene import BeeGraphicsScene +from ..base import BeeTestCase + + +class SQLiteIOTestCase(BeeTestCase): + + def setUp(self): + self.io = SQLiteIO(':memory:', None, create_new=True) + + def test_ẁrite_meta_application_id(self): + self.io.write_meta() + result = self.io.fetchone('PRAGMA application_id') + assert result[0] == SQLiteIO.APPLICATION_ID + + def test_ẁrite_meta_user_version(self): + self.io.write_meta() + result = self.io.fetchone('PRAGMA user_version') + assert result[0] == SQLiteIO.USER_VERSION + + def test_ẁrite_meta_foreign_keys(self): + self.io.write_meta() + result = self.io.fetchone('PRAGMA foreign_keys') + assert result[0] == 1 + + def test_create_schema_on_new_when_create_new(self): + self.io.create_schema_on_new() + result = self.io.fetchone( + 'SELECT COUNT(*) FROM sqlite_master ' + 'WHERE type="table" AND name NOT LIKE "sqlite_%"') + assert result[0] == 2 + + def test_create_schema_on_new_when_not_create_new(self): + self.io.create_new = False + self.io.create_schema_on_new() + result = self.io.fetchone( + 'SELECT COUNT(*) FROM sqlite_master ' + 'WHERE type="table" AND name NOT LIKE "sqlite_%"') + assert result[0] == 0 + + +class SQLiteIOWriteTestCase(BeeTestCase): + + def setUp(self): + self.scene = BeeGraphicsScene(None) + self.io = SQLiteIO(':memory:', self.scene, create_new=True) + + def test_calls_create_schema_on_new(self): + with patch.object(self.io, 'create_schema_on_new') as crmock: + with patch.object(self.io, 'fetchall'): + with patch.object(self.io, 'exmany'): + self.io.write() + crmock.assert_called_once() + + def test_calls_write_meta(self): + with patch.object(self.io, 'write_meta') as metamock: + with patch.object(self.io, 'fetchall'): + with patch.object(self.io, 'exmany'): + self.io.write() + metamock.assert_called_once() + + def test_inserts_new_item(self): + item = BeePixmapItem(QtGui.QImage(), filename='bee.png') + item.setScale(1.3) + item.setPos(44, 55) + item.pixmap_to_bytes = MagicMock(return_value=b'abc') + self.scene.addItem(item) + self.io.write() + + assert item.save_id == 1 + result = self.io.fetchone( + 'SELECT pos_x, pos_y, scale, filename, imgdata.data, type ' + 'FROM items ' + 'INNER JOIN imgdata on imgdata.item_id = items.id') + assert result[0] == 44.0 + assert result[1] == 55.0 + assert result[2] == 1.3 + assert result[3] == 'bee.png' + assert result[4] == b'abc' + assert result[5] == 'pixmap' + + def test_updates_existing_item(self): + item = BeePixmapItem(QtGui.QImage(), filename='bee.png') + item.setScale(1.3) + item.setPos(44, 55) + item.save_id = 1 + self.scene.addItem(item) + item.pixmap_to_bytes = MagicMock(return_value=b'abc') + self.io.write() + + item.setScale(0.7) + item.setPos(20, 30) + item.filename = 'new.png' + item.pixmap_to_bytes.return_value = b'updated' + self.io.create_new = False + self.io.write() + + assert self.io.fetchone('SELECT COUNT(*) from items') == (1,) + result = self.io.fetchone( + 'SELECT pos_x, pos_y, scale, filename, imgdata.data ' + 'FROM items ' + 'INNER JOIN imgdata on imgdata.item_id = items.id') + assert result[0] == 20 + assert result[1] == 30 + assert result[2] == 0.7 + assert result[3] == 'new.png' + assert result[4] == b'abc' + + def test_removes_nonexisting_item(self): + item = BeePixmapItem(QtGui.QImage(), filename='bee.png') + item.setScale(1.3) + item.setPos(44, 55) + self.scene.addItem(item) + self.io.write() + + self.scene.removeItem(item) + self.io.create_new = False + self.io.write() + + assert self.io.fetchone('SELECT COUNT(*) from items') == (0,) + assert self.io.fetchone('SELECT COUNT(*) from imgdata') == (0,) + + +class SQLiteIOLOadTestCase(BeeTestCase): + + def setUp(self): + self.scene = BeeGraphicsScene(None) + self.io = SQLiteIO(':memory:', self.scene, create_new=True) + + def test_loads(self): + root = os.path.dirname(__file__) + filename = os.path.join(root, '..', 'assets', 'test3x3.png') + with open(filename, 'rb') as f: + imgdata = f.read() + self.io.create_schema_on_new() + self.io.ex( + 'INSERT INTO items (type, pos_x, pos_y, scale, filename) ' + 'VALUES (?, ?, ?, ?, ?) ', + ('pixmap', 22.2, 33.3, 3.4, 'bee.png')) + self.io.ex('INSERT INTO imgdata (item_id, data) VALUES (?, ?)', + (1, imgdata)) + self.io.read() + assert len(self.scene.items()) == 1 + item = self.scene.items()[0] + assert item.pos().x() == 22.2 + assert item.pos().y() == 33.3 + assert item.scale_factor == 3.4 + assert item.filename == 'bee.png' + assert item.width == 3 + assert item.height == 3 diff --git a/tests/test_bee_json.py b/tests/test_bee_json.py deleted file mode 100644 index b483654..0000000 --- a/tests/test_bee_json.py +++ /dev/null @@ -1,30 +0,0 @@ -import os.path - -from PyQt6 import QtGui - -from beeref import bee_json -from beeref.items import BeePixmapItem -from .base import BeeTestCase - - -class BeeJsonTestCase(BeeTestCase): - - def test_dumps_loads(self): - root = os.path.dirname(__file__) - filename = os.path.join(root, 'assets', 'test3x3.png') - item = BeePixmapItem(QtGui.QImage(filename), filename) - item.setScale(2) - item.setPos(100, 200) - item.setZValue(3) - dump = bee_json.dumps({'items': [item]}) - - obj = bee_json.loads(dump) - assert len(obj['items']) == 1 - obj_item = obj['items'][0] - assert obj_item.scale_factor == 2 - assert obj_item.zValue() == 3 - assert obj_item.pos().x() == 100 - assert obj_item.pos().y() == 200 - assert obj_item.width == item.width - assert obj_item.height == item.height - assert obj_item.filename == filename diff --git a/tests/test_items.py b/tests/test_items.py index 7e37127..6bc0067 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -46,64 +46,3 @@ class BeePixmapItemTestCase(BeeTestCase): item.set_pos_center(0, 0) assert item.pos().x() == -100 assert item.pos().y() == -50 - - -class BeePixmapItemToBeeJsonTestCase(BeeTestCase): - - def test_basic(self): - item = BeePixmapItem(QtGui.QImage(), 'bee.png') - assert item.to_bee_json() == { - 'cls': 'BeePixmapItem', - 'scale': 1, - 'pos': [0.0, 0.0], - 'z': 0.0, - 'pixmap': '', - 'filename': 'bee.png', - } - - def test_scale(self): - item = BeePixmapItem(QtGui.QImage()) - item.setScale(2) - beejson = item.to_bee_json() - assert beejson['scale'] == 2 - - def test_position(self): - item = BeePixmapItem(QtGui.QImage()) - item.setPos(100, 200) - beejson = item.to_bee_json() - assert beejson['pos'] == [100, 200] - - def test_z_value(self): - item = BeePixmapItem(QtGui.QImage()) - item.setZValue(3) - beejson = item.to_bee_json() - assert beejson['z'] == 3.0 - - def test_pixmap(self): - root = os.path.dirname(__file__) - filename = os.path.join(root, 'assets', 'test3x3.png') - item = BeePixmapItem(QtGui.QImage(filename)) - beejson = item.to_bee_json() - assert len(beejson['pixmap']) > 0 - - -class BeePixmapItemFromBeeJsonTestCase(BeeTestCase): - - def test_basic(self): - bee_json = { - 'cls': 'BeePixmapItem', - 'scale': 2, - 'pos': [100.0, 200.0], - 'z': 3.0, - 'pixmap': '', - 'filename': 'bee.png', - } - - item = BeePixmapItem.from_bee_json(bee_json) - - assert isinstance(item, BeePixmapItem) - assert item.scale_factor == 2 - assert item.pos().x() == 100.0 - assert item.pos().y() == 200.0 - assert item.zValue() == 3.0 - assert item.filename == 'bee.png'