From 678461c001432829cd7c9d6cab5a37a486d4e15c Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Sun, 20 Nov 2022 21:08:42 +0900 Subject: [PATCH] Python: extract reference tags Signed-off-by: Masatake YAMATO --- Tmain/list-roles.d/stdout-expected.txt | 2 + Units/parser-python.r/reftag.d/args.ctags | 4 ++ Units/parser-python.r/reftag.d/expected.tags | 22 +++++++ Units/parser-python.r/reftag.d/input.py | 18 ++++++ parsers/python.c | 65 +++++++++++++++++--- 5 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 Units/parser-python.r/reftag.d/args.ctags create mode 100644 Units/parser-python.r/reftag.d/expected.tags create mode 100644 Units/parser-python.r/reftag.d/input.py diff --git a/Tmain/list-roles.d/stdout-expected.txt b/Tmain/list-roles.d/stdout-expected.txt index 7b4dc770b2..0a9aae6382 100644 --- a/Tmain/list-roles.d/stdout-expected.txt +++ b/Tmain/list-roles.d/stdout-expected.txt @@ -87,6 +87,7 @@ Python i/module indirectlyImported on module imported in a Python i/module namespace on namespace from where classes/variables/functions are imported Python x/unknown imported on imported from the other module Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python x/unknown ref off referenced anyhow R l/library library on library attached by library function R l/library require on library attached by require function R s/source source on source loaded by source fucntion @@ -208,6 +209,7 @@ Python i/module indirectlyImported on module imported in a Python i/module namespace on namespace from where classes/variables/functions are imported Python x/unknown imported on imported from the other module Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python x/unknown ref off referenced anyhow R l/library library on library attached by library function R l/library require on library attached by require function R s/source source on source loaded by source fucntion diff --git a/Units/parser-python.r/reftag.d/args.ctags b/Units/parser-python.r/reftag.d/args.ctags new file mode 100644 index 0000000000..d4d57f8efc --- /dev/null +++ b/Units/parser-python.r/reftag.d/args.ctags @@ -0,0 +1,4 @@ +--sort=no +--extras=+r +--fields=+r +--roles-Python.{unknown}=+{ref} diff --git a/Units/parser-python.r/reftag.d/expected.tags b/Units/parser-python.r/reftag.d/expected.tags new file mode 100644 index 0000000000..c54c23c567 --- /dev/null +++ b/Units/parser-python.r/reftag.d/expected.tags @@ -0,0 +1,22 @@ +a.b input.py /^import a.b$/;" i roles:imported +object input.py /^class X(object):$/;" x roles:ref +X input.py /^class X(object):$/;" c roles:def +self input.py /^ def __init__(self, n):$/;" x roles:ref +n input.py /^ def __init__(self, n):$/;" x roles:ref +__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def +self input.py /^ self.n = n$/;" x roles:ref +n input.py /^ self.n = n$/;" x roles:ref +n input.py /^ self.n = n$/;" x roles:ref +self input.py /^ def val(self):$/;" x roles:ref +val input.py /^ def val(self):$/;" m class:X roles:def +self input.py /^ return self.n$/;" x roles:ref +n input.py /^ return self.n$/;" x roles:ref +one input.py /^def one():$/;" f roles:def +X input.py /^ return X(1)$/;" x roles:ref +two input.py /^def two():$/;" f roles:def +X input.py /^ return X(2)$/;" x roles:ref +print input.py /^print (one().val() + two().val())$/;" x roles:ref +one input.py /^print (one().val() + two().val())$/;" x roles:ref +val input.py /^print (one().val() + two().val())$/;" x roles:ref +two input.py /^print (one().val() + two().val())$/;" x roles:ref +val input.py /^print (one().val() + two().val())$/;" x roles:ref diff --git a/Units/parser-python.r/reftag.d/input.py b/Units/parser-python.r/reftag.d/input.py new file mode 100644 index 0000000000..0136a53227 --- /dev/null +++ b/Units/parser-python.r/reftag.d/input.py @@ -0,0 +1,18 @@ +import a.b + +class X(object): + def __init__(self, n): + self.n = n + + def val(self): + return self.n + +def one(): + return X(1) + +def two(): + return X(2) + +print (one().val() + two().val()) + + diff --git a/parsers/python.c b/parsers/python.c index d5b75eed50..4f25390f08 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -96,6 +96,7 @@ typedef enum { } pythonModuleRole; typedef enum { + PYTHON_UNKNOWN_REFERENCED, PYTHON_UNKNOWN_IMPORTED, PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, } pythonUnknownRole; @@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = { }; static roleDefinition PythonUnknownRoles [] = { + { false,"ref", "referenced anyhow" }, { true, "imported", "imported from the other module" }, { true, "indirectlyImported", "classes/variables/functions/modules imported in alternative name" }, @@ -201,6 +203,7 @@ typedef struct { int indent; unsigned long lineNumber; MIOPos filePosition; + int reftag; } tokenInfo; struct pythonNestingLevelUserData { @@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident, return ACCESS_PROTECTED; } -static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token, +static void useTokenAsPartOfTag (tokenInfo *const token) +{ + if (token->reftag == CORK_NIL) + return; + + markCorkEntryAsPlaceholder (token->reftag, true); + token->reftag = CORK_NIL; +} + +static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token, const pythonKind kind) { accessType access; int parentKind = -1; NestingLevel *nl; + useTokenAsPartOfTag(token); initTagEntry (e, vStringValue (token->string), kind); e->lineNumber = token->lineNumber; @@ -284,7 +297,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token e->isFileScope = true; } -static int makeClassTag (const tokenInfo *const token, +static int makeClassTag (tokenInfo *const token, const vString *const inheritance, const vString *const decorators) { @@ -307,7 +320,7 @@ static int makeClassTag (const tokenInfo *const token, return CORK_NIL; } -static int makeFunctionTag (const tokenInfo *const token, +static int makeFunctionTag (tokenInfo *const token, const vString *const arglist, const vString *const decorators) { @@ -331,7 +344,7 @@ static int makeFunctionTag (const tokenInfo *const token, return CORK_NIL; } -static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind) +static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind) { if (PythonKinds[kind].enabled) { @@ -344,7 +357,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k return CORK_NIL; } -static int makeSimplePythonRefTag (const tokenInfo *const token, +static int makeSimplePythonRefTag (tokenInfo *const token, const vString *const altName, pythonKind const kind, int roleIndex, xtagType xtag) @@ -354,6 +367,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token, { tagEntryInfo e; + useTokenAsPartOfTag(token); initRefTagEntry (&e, vStringValue (altName ? altName : token->string), kind, roleIndex); @@ -393,6 +407,7 @@ static void clearPoolToken (void *data) token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); vStringClear (token->string); + token->reftag = CORK_NIL; } static void copyToken (tokenInfo *const dest, const tokenInfo *const src) @@ -403,6 +418,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src) dest->keyword = src->keyword; dest->indent = src->indent; vStringCopy(dest->string, src->string); + dest->reftag = src->reftag; } /* Skip a single or double quoted string. */ @@ -479,7 +495,7 @@ static void ungetToken (tokenInfo *const token) copyToken (NextToken, token); } -static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces) { int c; int n; @@ -694,6 +710,29 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) } } +static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +{ + readTokenFullNoRefTag (token, inclWhitespaces); + + if (token->type == TOKEN_IDENTIFIER + /* Don't make a ref tag for a number. */ + && (vStringLength(token->string) > 0 && + !isdigit((unsigned char)vStringChar(token->string, 0))) + && PythonKinds[K_UNKNOWN].enabled + && PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled) + { + const bool in_subparser = (Lang_python != getInputLanguage()); + if (in_subparser) + pushLanguage(Lang_python); + + token->reftag = makeSimpleRefTag (token->string, + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + + if (in_subparser) + popLanguage(); + } +} + static void readToken (tokenInfo *const token) { readTokenFull (token, false); @@ -788,6 +827,10 @@ static void readQualifiedName (tokenInfo *const nameToken) vString *qualifiedName = vStringNew (); tokenInfo *token = newToken (); + unsigned long lineNumber = nameToken->lineNumber; + MIOPos filePosition = nameToken->filePosition; + useTokenAsPartOfTag(token); + while (nameToken->type == TOKEN_IDENTIFIER || nameToken->type == '.') { @@ -795,6 +838,7 @@ static void readQualifiedName (tokenInfo *const nameToken) copyToken (token, nameToken); readToken (nameToken); + useTokenAsPartOfTag (nameToken); } /* put the last, non-matching, token back */ ungetToken (nameToken); @@ -805,6 +849,13 @@ static void readQualifiedName (tokenInfo *const nameToken) deleteToken (token); vStringDelete (qualifiedName); + + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (nameToken->string), + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + e.lineNumber = lineNumber; + e.filePosition = filePosition; + nameToken->reftag = makeTagEntry (&e); } } @@ -1490,7 +1541,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind) do { - const tokenInfo *const nameToken = nameTokens[i]; + tokenInfo *const nameToken = nameTokens[i]; vString **type = &(nameTypes[i++]); readToken (token);