Skip to content

Commit

Permalink
Add transformer to get back to an edifact file
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevincarrogan
kevincarrogan committed Jan 3, 2025
1 parent b48c1e9 commit e6f2a53
Show file tree
Hide file tree
Showing 3 changed files with 205 additions and 20 deletions.
26 changes: 17 additions & 9 deletions edifact/grammars/usage_data.lark
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@ file_header: _LINE_NUMBER \
_BACKSLASH \
"fileHeader" \
_BACKSLASH \
"CHIEF" \
SOURCE_SYSTEM \
_BACKSLASH \
"SPIRE" \
DESTINATION_SYSTEM \
_BACKSLASH \
"usageData" \
DATA_ID \
_BACKSLASH \
TIMESTAMP \
CREATION_DATE_TIME \
_BACKSLASH \
RUN_NUMBER \
_BACKSLASH
_BACKSLASH \
RESET_RUN_NUM?
file_trailer: _LINE_NUMBER \
_BACKSLASH \
"fileTrailer" \
Expand All @@ -25,12 +26,13 @@ licence_usage_transaction_header: _LINE_NUMBER \
_BACKSLASH \
TRANSACTION_REF \
_BACKSLASH \
"insert" \
ACTION \
_BACKSLASH \
LICENCE_REF \
_BACKSLASH \
LICENCE_STATUS \
_BACKSLASH
_BACKSLASH \
COMPLETION_DATE?
licence_usage_transaction_trailer: _LINE_NUMBER \
_BACKSLASH \
"end" \
Expand Down Expand Up @@ -100,15 +102,15 @@ LICENCE_USAGE_COUNT: NUMBER
_BACKSLASH: "\\"
TRANSACTION_REF: "LU" RUN_NUMBER "/" TRANSACTION_SEQUENCE_NUMBER
TRANSACTION_SEQUENCE_NUMBER: NUMBER
LICENCE_STATUS: ("C"|"E"|"O"|"S"|"D")
LICENCE_STATUS: "C" | "E" | "O" | "S" | "D"
COMPLETION_DATE: TIMESTAMP
RECORD_COUNT: NUMBER
LICENCE_REF: (LETTER | DIGIT | "/" | "-")+
LINE_NUM: NUMBER
CURRENCY: LETTER ~ 3
QUANTITY_USED: (NUMBER|FLOAT)
VALUE_USED: (NUMBER|FLOAT)
USAGE_TYPE: ("A"|"C"|"L"|"M"|"O")
USAGE_TYPE: "A" | "C" | "L" | "M" | "O"
DECLARATION_UCR: (LETTER | DIGIT | "/" | "-")+
DECLARATION_PART_NUM: (LETTER | DIGIT | "/" | "-") ~ 1..4
CONTROL_DATE: DATE
Expand All @@ -121,6 +123,12 @@ CLAIM_REF: WORD
CONSIGNEE_NAME: WORD+
DECLARATION_MRN: (LETTER | DIGIT) ~ 18
DEPARTURE_ICS: DIGIT ~ 2
ACTION: "insert"
SOURCE_SYSTEM: "CHIEF"
DESTINATION_SYSTEM: "SPIRE"
DATA_ID: "usageData"
CREATION_DATE_TIME: TIMESTAMP
RESET_RUN_NUM: "Y" | "N"

%import common.LETTER
%import common.INT -> NUMBER
Expand Down
71 changes: 67 additions & 4 deletions edifact/tests/test_visitors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from lark import Token, Tree

from edifact.parsers import usage_data_parser
from edifact.visitors import JsonPayload, RunNumberUpdater, SourceSplitter, TransactionMapper
from edifact.visitors import Edifact, JsonPayload, RunNumberUpdater, SourceSplitter, TransactionMapper
from mail.enums import SourceEnum
from mail.models import GoodIdMapping, LicenceIdMapping, LicencePayload, TransactionMapping
from mail.tests.factories import UsageDataFactory
Expand All @@ -26,14 +26,24 @@ def test_updates_run_number(self):
expected = Tree(
Token("RULE", "file"),
[
Tree(Token("RULE", "file_header"), [Token("TIMESTAMP", "201901130300"), Token("RUN_NUMBER", "12345")]),
Tree(
Token("RULE", "file_header"),
[
Token("SOURCE_SYSTEM", "CHIEF"),
Token("DESTINATION_SYSTEM", "SPIRE"),
Token("DATA_ID", "usageData"),
Token("CREATION_DATE_TIME", "201901130300"),
Token("RUN_NUMBER", "12345"),
],
),
Tree(
Token("RULE", "licence_usage_transaction"),
[
Tree(
Token("RULE", "licence_usage_transaction_header"),
[
Token("TRANSACTION_REF", "LU04148/00001"),
Token("ACTION", "insert"),
Token("LICENCE_REF", "GBOIE2017/12345B"),
Token("LICENCE_STATUS", "O"),
],
Expand Down Expand Up @@ -102,14 +112,24 @@ def test_only_returns_spire_lines(self):
expected = Tree(
Token("RULE", "file"),
[
Tree(Token("RULE", "file_header"), [Token("TIMESTAMP", "201901130300"), Token("RUN_NUMBER", "49543")]),
Tree(
Token("RULE", "file_header"),
[
Token("SOURCE_SYSTEM", "CHIEF"),
Token("DESTINATION_SYSTEM", "SPIRE"),
Token("DATA_ID", "usageData"),
Token("CREATION_DATE_TIME", "201901130300"),
Token("RUN_NUMBER", "49543"),
],
),
Tree(
Token("RULE", "licence_usage_transaction"),
[
Tree(
Token("RULE", "licence_usage_transaction_header"),
[
Token("TRANSACTION_REF", "LU04148/00001"),
Token("ACTION", "insert"),
Token("LICENCE_REF", "GBOIE2017/SPIRE"),
Token("LICENCE_STATUS", "O"),
],
Expand Down Expand Up @@ -176,14 +196,24 @@ def test_only_returns_lite_lines(self):
expected = Tree(
Token("RULE", "file"),
[
Tree(Token("RULE", "file_header"), [Token("TIMESTAMP", "201901130300"), Token("RUN_NUMBER", "49543")]),
Tree(
Token("RULE", "file_header"),
[
Token("SOURCE_SYSTEM", "CHIEF"),
Token("DESTINATION_SYSTEM", "SPIRE"),
Token("DATA_ID", "usageData"),
Token("CREATION_DATE_TIME", "201901130300"),
Token("RUN_NUMBER", "49543"),
],
),
Tree(
Token("RULE", "licence_usage_transaction"),
[
Tree(
Token("RULE", "licence_usage_transaction_header"),
[
Token("TRANSACTION_REF", "LU04148/00002"),
Token("ACTION", "insert"),
Token("LICENCE_REF", "GBSIE2017/LITE"),
Token("LICENCE_STATUS", "O"),
],
Expand Down Expand Up @@ -370,3 +400,36 @@ def test_json_payload(self):
}

self.assertEqual(payload, expected_payload)


class EditfactTests(TestCase):
def test_edifact(self):
file = """1\\fileHeader\\CHIEF\\SPIRE\\usageData\\201901130300\\49543\\
2\\licenceUsage\\LU04148/00005\\insert\\GBOGE2011/56789\\O\\
3\\line\\2\\17\\0\\
4\\usage\\O\\9GB000004988000-4750437112345\\G\\20190111\\0\\0\\\\000104\\\\\\\\\\\\\\
5\\usage\\O\\9GB000004988000-4750436912345\\Y\\20190111\\0\\0\\\\000104\\\\\\\\\\\\\\
6\\end\\line\\4
7\\end\\licenceUsage\\6
8\\licenceUsage\\LU04148/00006\\insert\\GBOGE2017/98765\\O\\
9\\line\\1\\0\\0\\
10\\usage\\O\\9GB000002816000-273993\\L\\20190109\\0\\0\\\\000316\\\\\\\\\\\\\\
11\\end\\line\\3
12\\end\\licenceUsage\\5
13\\licenceUsage\\LU04148/00007\\insert\\GBOGE2015/87654\\O\\
14\\line\\1\\1000000\\0\\GBP
15\\usage\\O\\9GB000003133000-784920212345\\E\\20190111\\0\\0\\\\000640\\\\\\\\\\\\\\
16\\usage\\O\\9GB000003133000-784918012345\\D\\20190111\\0\\0\\\\000640\\\\\\\\\\\\\\
17\\end\\line\\4
18\\end\\licenceUsage\\6
19\\licenceUsage\\LU04148/00008\\insert\\GBOGE2015/87654\\E\\
20\\line\\1\\9999\\0\\GBP
21\\usage\\O\\9GB000003333333-784920212345\\E\\20190111\\0\\0\\\\000640\\\\\\\\\\\\\\
22\\end\\line\\4
23\\end\\licenceUsage\\6
24\\fileTrailer\\4"""

tree = usage_data_parser.parse(file)
edifact = Edifact().transform(tree)

self.assertEqual(file, edifact)
128 changes: 121 additions & 7 deletions edifact/visitors.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import itertools

from lark import Discard, Token
from lark.visitors import Transformer, Visitor, v_args

Expand All @@ -23,8 +25,8 @@ def __init__(self, desired_source, *args, **kwargs):
super().__init__(*args, **kwargs)

def licence_usage_transaction(self, tree):
licence_usage_transaction_header, _, _ = tree.children
_, licence_ref, _ = licence_usage_transaction_header.children
licence_usage_transaction_header = tree.children[0]
licence_ref = licence_usage_transaction_header.children[2]

source = id_owner(licence_ref)
if not source == self.desired_source:
Expand All @@ -45,11 +47,9 @@ def __init__(self, usage_data, *args, **kwargs):
self.usage_data = usage_data
super().__init__(*args, **kwargs)

def visit(self, *args, **kwargs):
raise NotImplementedError("This should only be called topdown")

def licence_usage_transaction_header(self, tree):
self.transaction_ref, self.licence_ref, _ = tree.children
self.transaction_ref = str(tree.children[0])
self.licence_ref = str(tree.children[2])

def licence_line_header(self, tree):
line_num, *_ = tree.children
Expand Down Expand Up @@ -96,7 +96,7 @@ def licence_usage_transaction(self, licence_usage_transaction_header, licence_li
licence_payload["goods"].append(licence_line)
return licence_payload

def licence_usage_transaction_header(self, licence_reference, licence_status_code, completion_date=""):
def licence_usage_transaction_header(self, action, licence_reference, licence_status_code, completion_date=""):
return licence_reference, licence_status_code, completion_date

def TRANSACTION_REF(self, *args):
Expand All @@ -123,3 +123,117 @@ def licence_usage(self, *args):

def licence_line_trailer(self, *args):
return Discard


@v_args(inline=True)
class Edifact(Transformer):
def _to_line(self, line_type, fields, tokens):
data = {token.type: str(token) for token in tokens}
data = "\\".join([data.get(field, "") for field in fields])
return f"\\{line_type}\\{data}"

def _flatten(self, lists):
if not isinstance(lists, list):
return [lists]
return list(itertools.chain.from_iterable(self._flatten(l) for l in lists))

def file(self, *args):
lines = self._flatten(list(args))
lines = [f"{line_number}{line}" for line_number, line in enumerate(lines, start=1)]
lines = "\n".join(lines)
return lines

def file_header(self, *args):
return self._to_line(
"fileHeader",
[
"SOURCE_SYSTEM",
"DESTINATION_SYSTEM",
"DATA_ID",
"CREATION_DATE_TIME",
"RUN_NUMBER",
"RESET_RUN_NUMBER",
],
args,
)

def file_trailer(self, *args):
return self._to_line(
"fileTrailer",
[
"LICENCE_USAGE_COUNT",
],
args,
)

def licence_usage_transaction_header(self, *args):
return self._to_line(
"licenceUsage",
[
"TRANSACTION_REF",
"ACTION",
"LICENCE_REF",
"LICENCE_STATUS",
"COMPLETION_DATE",
],
args,
)

def licence_usage_transaction(self, *args):
return self._flatten(list(args))

def licence_usage_transaction_trailer(self, *args):
return self._to_line(
"end\\licenceUsage",
[
"RECORD_COUNT",
],
args,
)

def licence_line_header(self, *args):
return self._to_line(
"line",
[
"LINE_NUM",
"QUANTITY_USED",
"VALUE_USED",
"CURRENCY",
],
args,
)

def licence_line(self, *args):
return self._flatten(list(args))

def licence_line_trailer(self, *args):
return self._to_line(
"end\\line",
[
"RECORD_COUNT",
],
args,
)

def licence_usage(self, *args):
return self._to_line(
"usage",
[
"USAGE_TYPE",
"DECLARATION_UCR",
"DECLARATION_PART_NUM",
"CONTROL_DATE",
"QUANTITY_USED",
"VALUE_USED",
"CURRENCY",
"TRADER_ID",
"CLAIM_REF",
"ORIGIN_COUNTRY",
"CUSTOMS_MIC",
"CUSTOMS_MESSAGE",
"CONSIGNEE_NAME",
"DECLARATION_MRN",
"DEPARTURE_ICS",
],
args,
)

0 comments on commit e6f2a53

Please sign in to comment.