commit a3e49bc55ff18ae99d6c1c07f8e35978e42cc802
parent 7af24a6a1ad7262219bb479aa5f8dbd592764909
Author: Carlosokumu <carlosokumu254@gmail.com>
Date: Wed, 25 Feb 2026 14:20:00 +0300
Merge branch 'lash/resolvers' into carlos/attachments
merge resolvers for attachments
Diffstat:
20 files changed, 662 insertions(+), 61 deletions(-)
diff --git a/dummy/doc/internals.texi b/dummy/doc/internals.texi
@@ -9,8 +9,6 @@ Signatures are calculated and embedded on three elements individually, while at
The following describes any transformation applied to the XML. Also, any @code{sig} elements are removed before .
-Where expendient, @url{https://dwarfstd.org/doc/Dwarf3.pdf,LEB128s} encoding is used for variable-length integer encoding.
-
@anchor{serialize_attachment}
@subsection Attachment
@@ -49,6 +47,8 @@ The lookup key descriptions below are enumerated. Each element in the list shoul
Any optional and undefined elements @emph{must} contain a @emph{null value} in the serialization.
+Where expendient, @url{https://dwarfstd.org/doc/Dwarf3.pdf,LEB128s} encoding is used for variable-length integer encoding.
+
@xref{store,Cache store} for more details.
@subsection Ledger
diff --git a/dummy/setup.cfg b/dummy/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = usawa
-version = 0.1.0
+version = 0.2.0
description = Signed, immutable accounting.
author = Louis Holbrook
author_email = dev@holbrook.no
@@ -25,6 +25,7 @@ include_package_data = True
python_requires = >= 3.7
packages =
usawa
+ usawa.resolve
usawa.runnable
[options.entry_points]
diff --git a/dummy/setup.py b/dummy/setup.py
@@ -16,6 +16,7 @@ setup(
"PyNaCl~=1.6.0",
"python-gnupg~=0.4.9",
"rencode~=1.0.8",
+ "hexathon~=0.1.7",
"varints@file://" + varints_dir,
],
)
diff --git a/dummy/tests/entry.py b/dummy/tests/entry.py
@@ -2,7 +2,6 @@ import logging
import datetime
import unittest
import os
-import copy
from usawa import EntryPart, Entry, DemoWallet, ACL, UnitIndex, Asset
from usawa.error import ACLError
@@ -29,7 +28,6 @@ class TestEntry(unittest.TestCase):
o = Entry(42, datetime.datetime.strptime('2025-11-11', '%Y-%m-%d'), parent=self.parent, ref=self.ref, description=self.description, tx_datereg=self.dtreg)
o.add_part(src, debit=True)
o.add_part(dst)
- oo = copy.deepcopy(o)
s = o.serialize()
o = Entry.deserialize(s)
diff --git a/dummy/tests/import.xml b/dummy/tests/import.xml
@@ -0,0 +1 @@
+<ledger xmlns="http://usawa.defalsify.org/" version="1"><topic>0070309216f993d0382b6c8631b0419e97b24c4afe7d7c8e0ea2a421004314cceecad48c25ca02d3e039e7bd78c230c19543a016d3726aaaf5367616f60a2e68</topic><generated>2026-02-18T11:54:40Z</generated><src>defalsify.org</src><units base="BTC"><unit sym="BTC"><precision>2</precision><exchange>1000000000</exchange></unit></units><identity keyid="3b54648d60bb8a5b9e84fa0057f79b3a5996e511682e80176dc948dcbff5a4fc" didtype="usawa"/><incoming serial="0"><real unit="BTC"><income>0</income><expense>112425000</expense><asset>112425000</asset><liability>0</liability></real><digest algo="sha512">00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000</digest><sig keyid="3b54648d60bb8a5b9e84fa0057f79b3a5996e511682e80176dc948dcbff5a4fc" type="ed25519">ff1d1b74503879f76f054abdbc8a97d4411c2b073718896342915265d548700a7b3df7cd9b609464c86ad8aeb199e7f0e22df4fb3ff1aa113e4d084bbece0002</sig></incoming><entry><data><parent>00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000</parent><ref>43109a71-fe86-4297-9dcb-599e804e6945</ref><serial>1</serial><date>2026-02-18</date><dateTimeRegistered>2026-02-18T11:49:47Z</dateTimeRegistered><description>aerhaerh</description><debit type="expense"><unit>BTC</unit><account>general</account><amount>-12500</amount></debit><credit type="asset"><unit>BTC</unit><account>general</account><amount>12500</amount></credit><attachment mime="text/plain"><digest algo="sha256">cb943e74e4a80a3579800ee2b5e6b665318088b5460920c1281c91599c6d3610</digest><filename>test.txt</filename></attachment></data><sig type="ed25519" keyid="3b54648d60bb8a5b9e84fa0057f79b3a5996e511682e80176dc948dcbff5a4fc">761b6e7da5901c6154c8e9b64357fa597b3540f1fc1d91a5ee219d083d959ea2707d59db5193b30e168cf27b6579b47a53742c0b838e0384ac5443f5ef885203</sig></entry><entry><data><parent>7f9857864dc1f64bbdd052dfe50671e45bddda9783f01746270bb557484ad591fa3a6f29f323b7207cbece8b949ed6bcc585298b94d9f4005b0bfce5c82228e7</parent><ref>1c2a0841-8ac6-4c08-8a27-132386c5e102</ref><serial>2</serial><date>2026-02-18</date><dateTimeRegistered>2026-02-18T11:50:23Z</dateTimeRegistered><description>ergihegohaerg</description><debit type="expense"><unit>BTC</unit><account>general</account><amount>-112412500</amount></debit><credit type="asset"><unit>BTC</unit><account>general</account><amount>112412500</amount></credit><attachment mime="text/plain"><digest algo="sha256">cb943e74e4a80a3579800ee2b5e6b665318088b5460920c1281c91599c6d3610</digest><filename>test.txt</filename></attachment></data><sig type="ed25519" keyid="3b54648d60bb8a5b9e84fa0057f79b3a5996e511682e80176dc948dcbff5a4fc">54c6d346627dd1e330d5d2886676dc87074e07e1666fb9273579b236dede9ce60fe9a33b34e098406364a99847d27a6c8a643c97e4b70ed53748673ca89a4902</sig></entry></ledger>
+\ No newline at end of file
diff --git a/dummy/tests/ledger.py b/dummy/tests/ledger.py
@@ -119,5 +119,36 @@ class TestLedger(unittest.TestCase):
lxml.etree.fromstring(v, parser)
+ def test_ledger_truncate(self):
+ s = 'FOO'
+ uidx = UnitIndex(s)
+ uidx.add('USD')
+ ledger = Ledger(uidx)
+ store = LedgerStore(self.store, ledger=ledger)
+ store.start()
+
+ wallet = DemoWallet()
+ ledger.set_wallet(wallet)
+ x = EntryPart(s, 'income', 'foo', 1337, debit=True)
+ y = EntryPart(s, 'asset', 'foo', 1337)
+ v = Entry(ledger.peek(), datetime.datetime.now(), parent=ledger.current())
+ v.add_part(x, debit=True)
+ v.add_part(y)
+ v.sign(wallet)
+ ledger.add_entry(v)
+
+ x = EntryPart(s, 'expense', 'bar̈́', 42, debit=True)
+ y = EntryPart(s, 'liability', 'bar', 42)
+ v = Entry(ledger.peek(), datetime.datetime.now(), parent=ledger.current())
+ v.add_part(x, debit=True)
+ v.add_part(y)
+ v.sign(wallet)
+ ledger.add_entry(v)
+ ledger.sign()
+
+ ledger.truncate()
+ self.assertEqual(ledger.serial, 2)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/dummy/tests/resolver.py b/dummy/tests/resolver.py
@@ -0,0 +1,110 @@
+import unittest
+import logging
+import tempfile
+import hashlib
+import shutil
+import os
+import datetime
+
+import lxml.etree
+
+from usawa import Ledger, UnitIndex, Entry, EntryPart, DemoWallet, ACL
+from usawa.resolve.fs import FSResolver
+from usawa.error import VerifyError
+
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+testdir = os.path.realpath(os.path.dirname(__file__))
+
+hash_of_foo = 'f7fbba6e0636f890e56fbbf3283e524c6fa3204ae298382d624741d0dc6638326e282c41be5e4254d8820772c5518a2c5a8c0c7f7eda19594a7eb539453e1ed7'
+hash_of_bar = 'd82c4eb5261cb9c8aa9855edd67d1bd10482f41529858d925094d173fa662aa91ff39bc5b188615273484021dfb16fd8284cf684ccf0fc795be3aa2fc1e6c181'
+
+class TestResolver(unittest.TestCase):
+
+ def setUp(self):
+ self.parent = b'\x00' * 64
+ self.path = tempfile.mkdtemp()
+ self.backend = FSResolver(self.path)
+ self.dtreg = datetime.datetime.now()
+
+
+ def tearDown(self):
+ shutil.rmtree(self.path)
+
+
+ def test_resolve_putget(self):
+ h = hashlib.sha512()
+ v = os.urandom(1337)
+ h.update(v)
+ k = h.digest()
+ self.backend.put(k, v)
+ r = self.backend.get(k)
+ self.assertEqual(r, v)
+
+ k_wrong = os.urandom(32)
+ with self.assertRaises(ValueError):
+ r = self.backend.get(k_wrong)
+
+
+ def test_resolve_get_evil(self):
+ fp = os.path.join(self.path, hash_of_foo)
+ f = open(fp, 'wb')
+ f.write(b'bar')
+ f.close()
+ with self.assertRaises(VerifyError):
+ self.backend.get(hash_of_foo)
+
+
+ def test_resolve_lookup(self):
+ uidx = UnitIndex('FOO')
+ wallet = DemoWallet()
+ ledger = Ledger(uidx, wallet=wallet, topic=bytes.fromhex(hash_of_foo))
+ dst = EntryPart('FOO', 'asset', 'foo', 1337)
+ src = EntryPart('FOO', 'income', 'foo', 1337, debit=True)
+ entry = Entry(1, datetime.datetime.strptime('2025-11-11', '%Y-%m-%d'), parent=self.parent, tx_datereg=self.dtreg)
+ entry.add_part(dst)
+ entry.add_part(src, debit=True)
+ entry.sign(wallet)
+ first_entry_key = self.backend.put_entry(entry, 'sha512')
+ ledger.add_entry(entry)
+
+ dst = EntryPart('FOO', 'expense', 'bar̈́', 42, debit=True)
+ src = EntryPart('FOO', 'liability', 'bar', 42)
+ entry = Entry(ledger.peek(), datetime.datetime.now(), parent=ledger.current())
+ entry.add_part(dst)
+ entry.add_part(src, debit=True)
+ entry.sign(wallet)
+ ledger.add_entry(entry)
+ last_entry_key = self.backend.put_entry(entry, 'sha512')
+ ledger.sign()
+
+ ledger.truncate(lookup='sha512')
+ logg.debug('after trunc {}'.format(ledger.lookup))
+
+ acl = ACL.from_wallet(wallet)
+ s = ledger.to_string(lookup='sha512')
+ logg.debug('ledgerstring {}'.format(s))
+ ledger = ledger.from_string(s, acl=acl)
+
+ self.backend.restore_ledger(ledger)
+
+# tree = ledger.to_tree(lookup='sha512')
+# s = lxml.etree.tostring(tree)
+# ledger = Ledger(uidx, wallet=wallet, topic=bytes.fromhex(hash_of_foo))
+#
+# k = self.backend.get(first_entry_key)
+# first_entry = Entry.from_string(k, uidx)
+# ledger.add_entry(first_entry)
+#
+# k = self.backend.get(last_entry_key)
+# last_entry = Entry.from_string(k, uidx)
+# ledger.add_entry(last_entry)
+#
+# tree = ledger.to_tree(lookup='sha512')
+# s_orig = lxml.etree.tostring(tree)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/dummy/tests/store.py b/dummy/tests/store.py
@@ -104,5 +104,22 @@ class TestStore(unittest.TestCase):
store.load(acl=acl)
+ def test_store_import(self):
+ fp = os.path.join(testdir, 'import.xml')
+ ledger = Ledger.from_file(fp)
+ store = LedgerStore(self.store, ledger)
+ store.put_all(store_assets=True)
+
+ # TODO: less hacky test, perhaps a ledger.rewind() to get to zero state with everything else intact?
+ topic = ledger.topic
+ uidx = ledger.uidx
+ acl = ledger.acl
+ ledger = Ledger(uidx, topic=topic, acl=acl)
+ store = LedgerStore(self.store, ledger)
+ store.load(acl=acl)
+ # TODO: improve this test
+ self.assertEqual(len(ledger.entries), 2)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/dummy/usawa/asset.py b/dummy/usawa/asset.py
@@ -184,7 +184,9 @@ class Asset:
logg.debug('asset read {} bytes from path {} mime {}'.format(c, src, o.mime))
- o.uuid = str(uuid.uuid4())
+ h = hashlib.sha1()
+ h.update(o.digest)
+ #o.uuid = str(uuid.uuid1(h.digest()))
o.extref = extref
o.description = description
@@ -226,8 +228,8 @@ class Asset:
if canon:
return tree
- if self.uuid != None:
- tree.set('uuid', self.uuid)
+ #if self.uuid != None:
+ # tree.set('uuid', self.uuid)
if self.extref != None:
o = lxml.etree.SubElement(tree, 'extref')
@@ -261,7 +263,7 @@ class Asset:
@staticmethod
def from_tree(tree):
o = Asset()
- o.uuid = tree.get('uuid')
+ #o.uuid = tree.get('uuid')
o.mime = tree.get('mime')
v = tree.find('digest', namespaces=nsmap()).text
o.digest = bytes.fromhex(v)
@@ -293,15 +295,11 @@ class Asset:
:rtype: list
"""
def to_list(self):
- d = [
- self.mime,
- self.enc,
- self.uuid,
- self.extref,
- self.slug,
- self.ext,
- self.description,
- ]
+ d = []
+ #for k in ['mime', 'uuid', 'slug', 'ext', 'description', 'extref', 'enc']:
+ for k in ['mime', 'slug', 'ext', 'description', 'extref', 'enc']:
+ v = getattr(self, k)
+ d.append(v)
return d
@@ -327,9 +325,12 @@ class Asset:
o = Asset()
v = rencode.loads(data)
i = 0
- for k in ['mime', 'enc', 'uuid', 'extref', 'slug', 'ext', 'description']:
- if v[i] != None:
- setattr(o, k, v[i].decode('utf-8'))
+ #for k in ['mime', 'uuid', 'slug', 'ext', 'description', 'extref', 'enc']:
+ for k in ['mime', 'slug', 'ext', 'description', 'extref', 'enc']:
+ vv = v[i]
+ if isinstance(vv, bytes):
+ vv = vv.decode('utf-8')
+ setattr(o, k, vv)
i += 1
if isinstance(digest, str):
digest = bytes.fromhex(digest)
diff --git a/dummy/usawa/data/schema.xsd b/dummy/usawa/data/schema.xsd
@@ -27,6 +27,7 @@
<xs:element name="real" type="Balance" minOccurs="1" maxOccurs="unbounded" />
<xs:element name="virt" type="Balance" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="digest" type="Digest" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="lookup" type="Lookup" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="sig" type="Signature" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
<xs:attribute name="serial" type="xs:nonNegativeInteger" />
@@ -132,6 +133,13 @@
<xs:attribute name="uuid" type="xs:string" />
</xs:complexType>
+ <xs:complexType name="ParentRef">
+ <xs:sequence>
+ <xs:element name="digest" type="xs:string" />
+ <xs:element name="lookup" type="Lookup" minOccurs="0" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+
<xs:complexType name="EntryData">
<xs:sequence>
<xs:element name="parent" type="xs:string" />
diff --git a/dummy/usawa/entry.py b/dummy/usawa/entry.py
@@ -94,7 +94,6 @@ class EntryPart:
o = lxml.etree.Element('amount')
o.text = str(self.amount)
- logg.debug('tree amount {} {}'.format(self.unit, o.text))
tree.append(o)
return tree
@@ -205,6 +204,8 @@ class Entry:
self.description = description
self.debit = []
self.credit = []
+ self.lookup = None
+ self.lookup_algo = None
"""Add an entry part to the entry.
@@ -285,19 +286,29 @@ class Entry:
entry = Entry(serial, dt, ref=ref, parent=parent, tx_datereg=dtreg, description=description, unitindex=unitindex)
src = EntryPart.from_tree(src_tree, debit=True)
dst = EntryPart.from_tree(dst_tree)
- entry.add_part(src, debit=True)
entry.add_part(dst)
+ entry.add_part(src, debit=True)
for v in o.findall('attachment', namespaces=nsmap()):
asset = Asset.from_tree(v)
entry.attach(asset)
+ o = tree.find('lookup')
+ if o != None:
+ self.lookup_algo = o.get('algo')
+ self.lookup = o.text
+
for sig in tree.iter(NSPREFIX + 'sig'):
entry.add_signature(sig.get('keyid'), bytes.fromhex(sig.text))
return entry
+ @staticmethod
+ def from_string(s, unitindex, min=0):
+ tree = lxml.etree.fromstring(s)
+ return Entry.from_tree(tree, unitindex, min=min)
+
"""Generate the simple data structure used for rencode serialization.
:returns: data structure
@@ -317,6 +328,7 @@ class Entry:
for v in self.attachment:
attach.append(v.get_digest(binary=True))
+ logg.debug('serializing with parent {}'.format(self.parent.hex()))
d = [
self.parent,
self.serial,
@@ -324,8 +336,8 @@ class Entry:
self.dtreg.strftime('%Y%m%d%H%M%S'),
self.dt.strftime('%Y%m%d'),
self.description,
- debit,
credit,
+ debit,
attach,
]
return d
@@ -351,15 +363,16 @@ class Entry:
@staticmethod
def deserialize(data):
v = rencode.loads(data)
- parent = v[0].hex()
+ #parent = v[0].hex()
+ parent = v[0]
serial = v[1]
ref = v[2].decode('utf-8')
date_reg = datetime.datetime.strptime(v[3].decode('utf-8'), '%Y%m%d%H%M%S')
date = datetime.datetime.strptime(v[4].decode('utf-8'), '%Y%m%d')
#unit = v[5].decode('utf-8')
description = v[5].decode('utf-8')
- src_data = v[6]
- dst_data = v[7]
+ dst_data = v[6]
+ src_data = v[7]
attach_data = v[8]
o = Entry(serial, date, ref=ref, description=description, parent=parent, tx_datereg=date_reg)
for v in src_data:
@@ -487,7 +500,18 @@ class Entry:
return entry
+
+ """Verify signature on entry.
+ At least one signature must be valid for one of the public keys in the wallet or ACL.
+
+ :param wallet: Wallet holding a public key to verify.
+ :type wallet: usawa.Wallet
+ :param acl: A collection of public keys to verify.
+ :type acl: usawa.ACL
+ :raises VerifyError: Invalid signature.
+ :raises ValueError: Neither wallet nor ACL supplied.
+ """
def verify(self, wallet=None, acl=None):
if wallet == None and acl == None:
raise ValueError('verify needs at least one of wallet or acl')
@@ -518,7 +542,7 @@ class Entry:
:returns: XML tree representing the entry.
:rtype: lxml.etree.Element
"""
- def to_tree(self, canon=False):
+ def to_tree(self, canon=False, lookup=None):
#tree = etree.Element('entry', type=self.typ)
tree = lxml.etree.Element(NSPREFIX + 'entry', nsmap=nsmap())
data = lxml.etree.Element('data')
@@ -570,9 +594,51 @@ class Entry:
o.text = self.sigs[k].hex()
tree.append(o)
+ if lookup:
+ (k, v) = self.get_lookup(lookup, tree=tree)
+ o = lxml.etree.Element('lookup')
+ o.set('algo', lookup)
+ o.text = v
+ data.append(o)
+
return tree
+ def to_string(self, canon=False, lookup=None):
+ tree = self.to_tree(canon=canon, lookup=lookup)
+ #return lxml.etree.canonicalize(tree).decode('utf-8')
+ return lxml.etree.tostring(tree).decode('utf-8')
+
+
+ def get_lookup(self, lookup, tree=None):
+ if tree == None:
+ tree = self.to_tree(lookup=False)
+ h = None
+ if lookup == 'sha512':
+ h = hashlib.sha512()
+ elif lookup == 'sha256':
+ h = hashlib.sha256()
+ else:
+ raise ValueError('invalid lookup algo')
+
+ b = lxml.etree.canonicalize(tree, strip_text="True", exclude_tags=['lookup'])
+ h.update(b.encode('utf-8'))
+
+ return (h.digest().hex(), b,)
+#
+#
+# def tree_sum(self, lookup):
+# tree = self.to_tree()
+# b = lxml.etree.tostring(tree)
+# h = None
+# if lookup == 'sha512':
+# h = hashlib.sha512()
+# elif lookup == 'sha256':
+# h = hashlib.sha256()
+# h.update(b)
+# return h.digest()
+
+
"""Generate canonical XML for signature material.
:return: Signature material.
@@ -581,7 +647,7 @@ class Entry:
"""
def canon(self):
tree = self.to_tree(canon=True)
- b = lxml.etree.canonicalize(tree, strip_text=True, exclude_tags=['sig'])
+ b = lxml.etree.canonicalize(tree, strip_text=True, exclude_tags=['sig', 'lookup'])
return b.encode('utf-8')
diff --git a/dummy/usawa/ledger.py b/dummy/usawa/ledger.py
@@ -140,6 +140,8 @@ class RunningTotal:
return RunningTotal(unit, asset=asset, liability=liability)
+
+
"""Generate an XML tree from the current state of the object.
The XML generated can be used as a "real" or "virt" sub-element of the ledger/incoming/ element.
@@ -241,6 +243,8 @@ class Ledger:
self.entries = {}
self.running = {}
self.wallet = None
+ self.lookup = None
+ self.lookup_algo = 'sha512'
for k in self.uidx.syms():
if self.running.get(k) != None:
@@ -325,9 +329,9 @@ class Ledger:
:todo: enable use of multiple "real" elements
:todo: deduplicate signature from wallet identity if already exists
"""
- def to_tree(self):
- self.serial = self.base_serial
- self.cur = self.base
+ def to_tree(self, lookup=None):
+ #self.serial = self.base_serial
+ #self.cur = self.base
tree = lxml.etree.XML('<ledger xmlns="http://usawa.defalsify.org/" version="{}"></ledger>'.format(XML_FORMAT_VERSION))
# generate topic
@@ -358,14 +362,15 @@ class Ledger:
tree.append(units_tree)
# identity entry for the key signing the ledger state.
- o = self.wallet.to_tree()
- tree.append(o)
+ if self.wallet != None:
+ o = self.wallet.to_tree()
+ tree.append(o)
# incoming state
incoming = lxml.etree.SubElement(tree, 'incoming')
# incoming serial
- incoming.set('serial', str(self.serial))
+ incoming.set('serial', str(self.base_serial))
# incoming base (real) currency balance
o = self.running[self.uidx.base].to_tree()
@@ -382,6 +387,13 @@ class Ledger:
o = lxml.etree.SubElement(incoming, 'digest')
o.attrib['algo'] = 'sha512'
o.text = self.base.hex()
+ incoming.append(o)
+
+ if self.lookup != None:
+ o = lxml.etree.SubElement(incoming, 'lookup')
+ o.attrib['algo'] = self.lookup_algo
+ o.text = self.lookup
+ incoming.append(o)
# incoming signatures
# sign the ledger if it has no signatures
@@ -391,6 +403,8 @@ class Ledger:
# apply all already existing signatures (e.g. from import)
for k in self.sigs.keys():
sig = self.sigs[k]
+ if len(sig) == 0:
+ continue
o = lxml.etree.SubElement(incoming, 'sig')
o.set('keyid', k.hex())
o.set('type', 'ed25519')
@@ -400,7 +414,7 @@ class Ledger:
# apply all entries in object state
for k in self.entries.keys():
v = self.entries[k]
- entry_tree = v.to_tree()
+ entry_tree = v.to_tree(lookup=lookup)
tree.append(entry_tree)
return tree
@@ -451,16 +465,21 @@ class Ledger:
:raises ValueError: When entry parent does not match ledger state.
:raises VerifyError: When entry is missing valid signature.
"""
- def add_entry(self, entry):
- if self.cur != entry.parent:
- raise ValueError('entry parent does not match ledger state')
+ def add_entry(self, entry, check_parent=True):
+ if check_parent and self.cur != entry.parent:
+ raise ValueError('entry parent {} does not match ledger state {}'.format(entry.parent.hex(), self.cur.hex()))
self.check_sigs(entry)
# update the internal state
self.serial = entry.serial
- oldsum = self.cur
- self.cur = entry.sum()[0]
- entry.parent = oldsum
+ #oldsum = self.cur
+ #self.cur = entry.sum()[0]
+ (k, v) = entry.get_lookup(self.lookup_algo)
+ logg.debug('addentr entry {} {}'.format(k, v))
+ #entry.parent = oldsum
+ entry.parent = self.cur
+ self.cur = bytes.fromhex(k)
+ logg.debug('selfcur is now {}'.format(self.cur.hex()))
self.apply_entryparts(entry)
# Add entry to the ledger object.
@@ -540,7 +559,7 @@ class Ledger:
public_key = bytes.fromhex(keyid)
wallet = DemoWallet(publickey=public_key)
ledger.set_wallet(wallet)
- logg.warn('currently only support for single identity')
+ logg.warning('currently only support for single identity')
break
o = part.find('real', namespaces=nsmap())
@@ -560,12 +579,26 @@ class Ledger:
if ledger.running.get(unit) == None:
ledger.running[unit] = RunningTotal(unit, unitindex)
+ o = part.find('lookup', namespaces=nsmap())
+ if o != None:
+ ledger.lookup = o.text
+ ledger.lookup_algo = o.get('algo')
+
ledger.apply_entries(tree)
logg.debug('loaded ledger tree last serial {}'.format(ledger.serial))
return ledger.check()
+ @staticmethod
+ def from_file(filepath, acl=None):
+ f = open(filepath, 'rb')
+ v = f.read()
+ f.close()
+ return Ledger.from_string(v, acl=acl)
+
+
+
"""Append all entries from XML tree to ledger.
:param tree: A parsed XML tree.
@@ -577,23 +610,45 @@ class Ledger:
i = 0
for v in tree.iter(NSPREFIX + 'entry'):
i += 1
- logg.debug('processing entry {}'.format(v))
+ logg.debug('>>>>>>>>>>>>> processing entry {}'.format(lxml.etree.tostring(v)))
o = Entry.from_tree(v, self.uidx, min=self.serial)
self.add_entry(o)
+ (k, v) = o.get_lookup('sha512')
if o.serial > last:
last = o.serial
+ #self.cur = k
if i > 0:
self.serial = last
logg.info('last entry from tree serial ' + str(self.serial))
+ def last_entry(self):
+ serial = -1
+ digest = None
+ for i in self.entries.keys():
+ if i > serial:
+ serial = i
+ return self.entries[serial]
+
+
"""Calculate and apply ledger state from current entries in the object, and remove entries.
After this call, the XML export will not contain any entry elements, and will have digest and serial from the last entry that existed in the ledger.
"""
- def truncate(self):
+ def truncate(self, lookup=None):
self.base = self.cur
self.base_serial = self.serial
+ logg.debug('base serial now {}'.format(self.base_serial))
+ try:
+ entry = self.last_entry()
+ except KeyError: # if no entries
+ self.entries = {}
+ return
+ if lookup != None:
+ (k, v) = entry.get_lookup(lookup)
+ logg.debug('trunc looup {} {}'.format(k, v))
+ self.lookup = k
+ self.lookup_algo = lookup
self.entries = {}
@@ -610,9 +665,15 @@ class Ledger:
:return: XML document in UTF-8 format.
:rtype: str
"""
- def to_string(self):
- tree = self.to_tree()
- return lxml.etree.tostring(tree)
+ def to_string(self, lookup=None):
+ tree = self.to_tree(lookup=lookup)
+ return lxml.etree.tostring(tree).decode('utf-8')
+
+
+ @staticmethod
+ def from_string(s, acl=None):
+ tree = lxml.etree.fromstring(s)
+ return Ledger.from_tree(tree, acl=acl)
"""Returns the digest of the current state of the ledger.
@@ -634,7 +695,7 @@ class Ledger:
:rtype: str
"""
def canon(self):
- tree = self.to_tree()
+ tree = self.to_tree(lookup=False)
b = lxml.etree.canonicalize(tree, strip_text=True, exclude_tags=['sig'])
return b.encode('utf-8')
diff --git a/dummy/usawa/resolve/__init__.py b/dummy/usawa/resolve/__init__.py
diff --git a/dummy/usawa/resolve/base.py b/dummy/usawa/resolve/base.py
@@ -0,0 +1,127 @@
+import hashlib
+import logging
+
+import hexathon
+
+from usawa import Entry
+from usawa.error import VerifyError
+from usawa.constant import DEFAULTPARENT
+
+logg = logging.getLogger('usawa.resolve')
+
+
+"""Verifies a key as a sha512 digest, optionally against the given value.
+
+:param k: Key to check.
+:type k: bytes or hex string
+:param v: Value to check against key.
+:type v: bytes
+:raises ValueError: Invalid key (not 512 bits)
+:raises VerifyError: Value digest does not match key.
+:return: Key as hex string
+:rtype: str
+"""
+def sha512_verify(k, v=None):
+ if isinstance(k, str):
+ k = bytes.fromhex(k)
+ if len(k) != 64:
+ raise ValueError('expect 512 bit key')
+ khx = hexathon.uniform(k.hex())
+ if v != None:
+ h = hashlib.sha512()
+ h.update(v)
+ if k != h.digest():
+ raise VerifyError(khx)
+ return khx
+
+
+class BaseResolver:
+
+ """A resolver abstracts an immutable store used for storing asset data.
+
+ Key/value pairs put to the store are checked by the verifier function before submission. Similarly, the value retrieved is checked by the verifier against the key used to get it.
+
+ :raises IOError: Resolver backend unavailable, temporarily or permanently.
+ :param verifier: Verifier function for checking keys and key/value relation.
+ :type verifier: function, by default usawa.resolve.sha512_verify
+ """
+ def __init__(self, verifier=sha512_verify):
+ self.verifier = verifier
+
+
+ """Get value for key.
+
+ :param k: Key
+ :type k: bytes or hex string
+ :raises FileNotFoundError: Key does not exist.
+ :raises PermissionError: No access to data.
+ :raises IOError: Any other read problem.
+ :return: Value
+ :rtype: bytes
+ """
+ def get(self, k):
+ raise NotImplementedError()
+
+ """Put value under key.
+
+ Value may not be available immediately after method returns, since the implementation may store asynchronously.
+
+ :param k: Key
+ :type k: bytes or hex string
+ :param v: Value
+ :type v: bytes
+ :raises FileExistsError: Key exists.
+ :raises PermissionError: No access to data.
+ :raises IOError: Any other read problem.
+ :return: A textual representation of the key
+ :rtype: str
+ """
+ def put(self, k, v):
+ raise NotImplementedError()
+
+
+ """Check availability of data without invoking a full get call.
+
+ If -1 is returned, the key does not exist.
+
+ If 0 is returned, the key has been added but may not yet be available for a get() call.
+
+ If a value greater than 0 is returned, get() can safely be called.
+
+ :param k: Key to check state for
+ :type k: bytes or hex string
+ :return: storage state
+ :rtype: int
+ """
+ def state(self, k):
+ raise NotImplementedError()
+
+
+ def put_entry(self, entry, lookup=None):
+ k = None
+ (k, v) = entry.sum()
+ self.put(k, v)
+ if lookup != None:
+ (k, v) = entry.get_lookup(lookup)
+ self.put(k, v.encode('utf-8'))
+ logg.debug('putentr entry {} {}'.format(k, v))
+ return k
+
+
+ def restore_ledger(self, ledger, min=0):
+ lookup = self.get(ledger.current())
+ while True:
+ entry = Entry.from_string(lookup, ledger.uidx)
+ if entry.serial == 0 or entry.serial < min:
+ break
+ logg.debug('restore entry {} {}'.format(str(entry), lookup))
+ k = entry.parent
+ ledger.add_entry(entry, check_parent=False)
+ if k == DEFAULTPARENT:
+ break
+ logg.debug('getting parent {}'.format(k.hex()))
+ lookup = self.get(k)
+ #v = self.get(k)
+ #logg.debug('getting parent {} {}'.format(k.hex(), v))
+ #entry_nolookup = Entry.from_string(v, ledger.uidx)
+ #lookup = self.get(entry_nolookup.lookup)
diff --git a/dummy/usawa/resolve/fs.py b/dummy/usawa/resolve/fs.py
@@ -0,0 +1,66 @@
+import os
+import logging
+
+from .base import BaseResolver, sha512_verify
+from usawa.error import VerifyError
+
+logg = logging.getLogger('usawa.resolve.fs')
+
+
+def normalize_keyname(k):
+ if isinstance(k, bytes):
+ k = k.hex()
+ else:
+ bytes.fromhex(k)
+ return hexathon.uniform(k)
+
+
+class FSResolver(BaseResolver):
+
+ """Resolver implementation for a filesystem directory.
+
+ If directory does not exist it will be created, along with any necessary ascendants.
+
+ :param path: Path to directory to store under.
+ :type path: str
+ :raises PermissionError: Insufficient access to create directory.
+ :seealso: usawa.resolve.BaseResolver
+ """
+ def __init__(self, path, verifier=sha512_verify):
+ super(FSResolver, self).__init__(verifier=verifier)
+ self.path = os.path.realpath(path)
+ os.makedirs(self.path, exist_ok=True)
+
+
+ """Implements usawa.resolve.BaseResolver
+ """
+ def get(self, k):
+ khx = self.verifier(k)
+ fp = os.path.join(self.path, khx)
+ f = open(fp, 'rb')
+ v = f.read()
+ f.close()
+ if not self.verifier(k, v):
+ raise VerifyError(khx)
+ return v
+
+
+ """Implements usawa.resolve.BaseResolver
+ """
+ def put(self, k, v):
+ khx = self.verifier(k, v=v)
+ fp = os.path.join(self.path, khx)
+ f = open(fp, 'wb')
+ c = f.write(v)
+ logg.debug('{} bytes written for key {}'.format(c, khx))
+ f.close()
+ return k
+
+
+ """Implements usawa.resolve.BaseResolver
+ """
+ def have(self, k):
+ r = -1
+ if os.path.is_file():
+ r = 1
+ return r
diff --git a/dummy/usawa/runnable/add.py b/dummy/usawa/runnable/add.py
@@ -29,6 +29,8 @@ class Context:
self.output = None
self.f = None
self.attach = []
+ self.valkey_host = None
+ self.valkey_port = None
def close(self):
@@ -38,10 +40,10 @@ class Context:
def open(self, output):
if output == '<stdout>':
- self.f = sys.stdout.buffer
+ self.f = sys.stdout
logg.debug('output is stdout')
else:
- self.f = open(output, 'wb')
+ self.f = open(output, 'w')
return self
@@ -72,6 +74,9 @@ class Context:
o = Asset.from_file(v)
ctx.attach.append(o)
+ ctx.valkey_host = args.valkey_host
+ ctx.valkey_port = args.valkey_port
+
return ctx
@@ -129,6 +134,9 @@ argp.add_argument('-d', '--description', dest='description', type=str, help='int
argp.add_argument('-u', '--unit', type=str, default=UnitIndex.default_unit, help='Unit to use for transaction')
argp.add_argument('--unit-precision', dest='unit_precision', type=int, default=UnitIndex.default_precision, help='Unit precision')
argp.add_argument('--unit-rate', dest='unit_precision', type=float, default=1.0, help='Unit exchange rate')
+argp.add_argument('--valkey-host', dest='valkey_host', type=str, default='localhost', help='Valkey host')
+argp.add_argument('--valkey-port', dest='valkey_port', type=int, default=6379, help='Valkey port')
+
argp.add_argument('ledger_xml_file', type=str, help='load ledger metadata from XML file')
arg = argp.parse_args()
ctx = Context.from_args(arg)
@@ -140,7 +148,7 @@ ledger_tree = load(arg.ledger_xml_file)
uidx = UnitIndex.from_tree(ledger_tree)
ledger = Ledger.from_tree(ledger_tree)
-db = ValkeyStore('')
+db = ValkeyStore('', host=ctx.valkey_host, port=ctx.valkey_port)
store = LedgerStore(db, ledger)
pk = store.get_key()
wallet = DemoWallet(privatekey=pk)
diff --git a/dummy/usawa/runnable/create.py b/dummy/usawa/runnable/create.py
@@ -33,10 +33,10 @@ class Context:
def open(self, output):
if output == '<stdout>':
- self.f = sys.stdout.buffer
+ self.f = sys.stdout
logg.debug('output is stdout')
else:
- self.f = open(output, 'wb')
+ self.f = open(output, 'w')
return self
diff --git a/dummy/usawa/runnable/import.py b/dummy/usawa/runnable/import.py
@@ -0,0 +1,73 @@
+import os
+import sys
+import logging
+import urllib.parse
+import argparse
+import uuid
+import datetime
+
+from usawa import Ledger, Entry, EntryPart, DemoWallet, load, ACL
+from usawa.constant import CATEGORIES
+from usawa.store import LedgerStore
+from whee.valkey import ValkeyStore
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class Context:
+
+ def __init__(self):
+ self.unit = None
+ self.uidx = None
+ self.output = None
+ self.f = None
+ self.valkey_host = None
+ self.valkey_port = None
+
+
+ def close(self):
+ if self.f and self.f != sys.stdout:
+ self.f.close()
+
+
+ def open(self, output):
+ if output == '<stdout>':
+ self.f = sys.stdout.buffer
+ logg.debug('output is stdout')
+ else:
+ self.f = open(output, 'wb')
+ return self
+
+ @staticmethod
+ def from_args(args):
+ ctx = Context()
+ if args.output != None:
+ ctx.output = os.path.realpath(args.output)
+ else:
+ ctx.output = '<stdout>'
+
+ ctx.valkey_host = args.valkey_host
+ ctx.valkey_port = args.valkey_port
+
+ return ctx
+
+
+argp = argparse.ArgumentParser()
+argp.add_argument('-o', type=str, dest='output', help='output file for resulting XML document')
+argp.add_argument('--valkey-host', dest='valkey_host', type=str, default='localhost', help='Valkey host')
+argp.add_argument('--valkey-port', dest='valkey_port', type=int, default=6379, help='Valkey port')
+argp.add_argument('ledger_xml_file', type=str, help='load ledger metadata from XML file')
+arg = argp.parse_args()
+ctx = Context.from_args(arg)
+
+ledger = Ledger.from_file(arg.ledger_xml_file)
+
+storedb = ValkeyStore('', host=ctx.valkey_host, port=ctx.valkey_port)
+store = LedgerStore(storedb, ledger)
+#pk = store.get_key()
+#wallet = DemoWallet(privatekey=pk)
+#acl = ACL.from_wallet(wallet)
+#store.load(acl=acl)
+store.put_all(store_assets=True)
+sys.stdout.write(ledger.to_string())
diff --git a/dummy/usawa/runnable/view.py b/dummy/usawa/runnable/view.py
@@ -22,6 +22,8 @@ class Context:
self.uidx = None
self.output = None
self.f = None
+ self.valkey_host = None
+ self.valkey_port = None
def close(self):
@@ -44,24 +46,31 @@ class Context:
ctx.output = os.path.realpath(args.output)
else:
ctx.output = '<stdout>'
+
+ ctx.valkey_host = args.valkey_host
+ ctx.valkey_port = args.valkey_port
+
return ctx
argp = argparse.ArgumentParser()
argp.add_argument('-o', type=str, dest='output', help='output file for resulting XML document')
+argp.add_argument('--valkey-host', dest='valkey_host', type=str, default='localhost', help='Valkey host')
+argp.add_argument('--valkey-port', dest='valkey_port', type=int, default=6379, help='Valkey port')
argp.add_argument('ledger_xml_file', type=str, help='load ledger metadata from XML file')
arg = argp.parse_args()
ctx = Context.from_args(arg)
-ledger = None
-ledger_tree = load(arg.ledger_xml_file)
-uidx = UnitIndex.from_tree(ledger_tree)
-ledger = Ledger.from_tree(ledger_tree)
+#ledger = None
+#ledger_tree = load(arg.ledger_xml_file)
+#uidx = UnitIndex.from_tree(ledger_tree)
+#ledger = Ledger.from_tree(ledger_tree)
+ledger = Ledger.from_file(arg.ledger_xml_file)
-storedb = ValkeyStore('')
+storedb = ValkeyStore('', host=ctx.valkey_host, port=ctx.valkey_port)
store = LedgerStore(storedb, ledger)
pk = store.get_key()
wallet = DemoWallet(privatekey=pk)
acl = ACL.from_wallet(wallet)
store.load(acl=acl)
-sys.stdout.buffer.write(ledger.to_string())
+sys.stdout.write(ledger.to_string())
diff --git a/dummy/usawa/store.py b/dummy/usawa/store.py
@@ -185,7 +185,7 @@ class LedgerStore(Interface):
#asset = Asset.deserialize(v, digest=o.get_digest(binary=True))
entry.attachment[i] = asset
i += 1
- entry.verify(acl=acl)
+ #entry.verify(acl=acl)
return entry
@@ -282,3 +282,25 @@ class LedgerStore(Interface):
"""
def get(self, k):
return self.__o.get(k)
+
+
+ """Store all entries in the ledger state.
+
+ Errors due to duplicate entry and asset insert attempts will be ignored.
+
+ :param store_assets: Add all attachment assets from each entry.
+ :type store_assets: boolean
+ :raises FileExistsError: If duplicate entry is found.
+ """
+ def put_all(self, store_assets=False):
+ for k in self.ledger.entries.keys():
+ entry = self.ledger.entries[k]
+ try:
+ self.add_entry(entry, update_ledger=False)
+ except FileExistsError as e:
+ logg.info('putall skip duplicate entry {}'.format(entry))
+ for asset in entry.attachment:
+ try:
+ self.add_asset(asset)
+ except FileExistsError:
+ logg.info('putall skip duplicate asset {}'.format(asset))
