diff --git a/Package.resolved b/Package.resolved index d8271840f..caf8c2de9 100644 --- a/Package.resolved +++ b/Package.resolved @@ -15,8 +15,17 @@ "repositoryURL": "https://github.com/Carthage/Commandant.git", "state": { "branch": null, - "revision": "7f29606ec3a2054a601f0e72f562a104dbc1a11a", - "version": "0.13.0" + "revision": "066bf9a79c37cf24fe4746ab7a1793b0359d1fe5", + "version": "0.14.0" + } + }, + { + "package": "CSQLite", + "repositoryURL": "https://github.com/groue/CSQLite.git", + "state": { + "branch": null, + "revision": "51210b121508dd91dcced13d398269c004b0f1b5", + "version": "0.2.0" } }, { @@ -24,8 +33,8 @@ "repositoryURL": "https://github.com/Quick/Nimble.git", "state": { "branch": null, - "revision": "21f4fed2052cea480f5f1d2044d45aa25fdfb988", - "version": "7.1.1" + "revision": "8023e3980d91b470ad073d6da843b73f2eeb1844", + "version": "7.1.2" } }, { diff --git a/Package.swift b/Package.swift index 0c8fd0539..880211157 100644 --- a/Package.swift +++ b/Package.swift @@ -27,12 +27,17 @@ let package = Package( dependencies: [ "SWXMLHash", "Yams", + "CSQLite" ], exclude: [ "clang-c", "sourcekitd.h", ] ), + .target( + name: "CSQLite", + dependencies: [] + ), .testTarget( name: "SourceKittenFrameworkTests", dependencies: [ diff --git a/Source/CSQLite/anchor.c b/Source/CSQLite/anchor.c new file mode 100755 index 000000000..dee6cc76c --- /dev/null +++ b/Source/CSQLite/anchor.c @@ -0,0 +1 @@ +/* Empty file so SPM will build this target */ diff --git a/Source/CSQLite/include/module.modulemap b/Source/CSQLite/include/module.modulemap new file mode 100755 index 000000000..0a291b5e2 --- /dev/null +++ b/Source/CSQLite/include/module.modulemap @@ -0,0 +1,5 @@ +module CSQLite [system] { + header "shim.h" + link "sqlite3" + export * +} diff --git a/Source/CSQLite/include/shim.h b/Source/CSQLite/include/shim.h new file mode 100644 index 000000000..c2252373c --- /dev/null +++ b/Source/CSQLite/include/shim.h @@ -0,0 +1,14 @@ +#ifndef CSQLITE_SHIM_H +#define CSQLITE_SHIM_H + +#include + +typedef void(*errorLogCallback)(void *pArg, int iErrCode, const char *zMsg); + +// Wrapper around sqlite3_config(SQLITE_CONFIG_LOG, ...) which is a variadic +// function that can't be used from Swift. +static inline void registerErrorLogCallback(errorLogCallback callback) { + sqlite3_config(SQLITE_CONFIG_LOG, callback, 0); +} + +#endif // CSQLITE_SHIM_H diff --git a/Source/SourceKittenFramework/Clang+SourceKitten.swift b/Source/SourceKittenFramework/Clang+SourceKitten.swift index d3078a263..ed937878e 100644 --- a/Source/SourceKittenFramework/Clang+SourceKitten.swift +++ b/Source/SourceKittenFramework/Clang+SourceKitten.swift @@ -32,7 +32,7 @@ private func setUUIDString(uidString: String, `for` file: String) { } struct ClangIndex { - private let index = clang_createIndex(0, 1) + public let index = clang_createIndex(0, 1) func open(file: String, args: [UnsafePointer?]) -> CXTranslationUnit { return clang_createTranslationUnitFromSourceFile(index, file, Int32(args.count), args, 0, nil)! @@ -71,10 +71,7 @@ extension CXCursor { } func extent() -> (start: SourceLocation, end: SourceLocation) { - let extent = clang_getCursorExtent(self) - let start = SourceLocation(clangLocation: clang_getRangeStart(extent)) - let end = SourceLocation(clangLocation: clang_getRangeEnd(extent)) - return (start, end) + return clang_getCursorExtent(self).range() } func shouldDocument() -> Bool { @@ -193,6 +190,30 @@ extension CXCursor { } return commentBody } + + func commentBodyExtent() -> (start: SourceLocation, end: SourceLocation) { + return clang_Cursor_getCommentRange(self).range() + } + + func tokensAndCursors(handle: (([CXToken], [CXCursor]) -> Void)) { + guard let translation = clang_Cursor_getTranslationUnit(self) else { return } + var token : CXToken = CXToken.init() + var numTokens : UInt32 = 0 + withUnsafeMutablePointer(to: &token) { (pointer) in + var p = Optional(pointer) + clang_tokenize(translation, clang_getCursorExtent(self), &p, &numTokens) + var tokens = Array(UnsafeBufferPointer.init(start: p, count: Int(numTokens))) + var cursors = Array.init(repeating: CXCursor.init(), count: Int(numTokens)) + //let cursorPointer = malloc(MemoryLayout.size(ofValue: CXCursor.self)*Int(numTokens)).assumingMemoryBound(to: CXCursor.self) + //var realPointer = UnsafeMutablePointer.init(mutating: (tokens.withUnsafeBufferPointer {return $0}).baseAddress) + clang_annotateTokens(translation, &tokens, numTokens, &cursors) + //let cursors = Array(UnsafeBufferPointer.init(start: cursorPointer, count: Int(numTokens))) + + handle(tokens, cursors) + + clang_disposeTokens(translation, p, numTokens) + } + } func swiftDeclarationAndName(compilerArguments: [String]) -> (swiftDeclaration: String?, swiftName: String?) { let file = location().file @@ -286,4 +307,13 @@ extension CXComment { } } +extension CXSourceRange { + func range() -> (start: SourceLocation, end: SourceLocation) { + let start = SourceLocation(clangLocation: clang_getRangeStart(self)) + let end = SourceLocation(clangLocation: clang_getRangeEnd(self)) + + return (start: start, end: end) + } +} + #endif diff --git a/Source/SourceKittenFramework/ClangTranslationUnit.swift b/Source/SourceKittenFramework/ClangTranslationUnit.swift index cc391c521..f6e437d09 100644 --- a/Source/SourceKittenFramework/ClangTranslationUnit.swift +++ b/Source/SourceKittenFramework/ClangTranslationUnit.swift @@ -61,7 +61,7 @@ public struct ClangTranslationUnit { let clangIndex = ClangIndex() clangTranslationUnits = headerFiles.map { clangIndex.open(file: $0, args: cStringCompilerArguments) } declarations = clangTranslationUnits - .flatMap { $0.cursor().compactMap({ SourceDeclaration(cursor: $0, compilerArguments: compilerArguments) }) } + .flatMap { $0.cursor().compactMap({ SourceDeclaration(cursor: $0, compilerArguments: compilerArguments, index: clangIndex.index) }) } .rejectEmptyDuplicateEnums() .distinct() .sorted() diff --git a/Source/SourceKittenFramework/File.swift b/Source/SourceKittenFramework/File.swift index b1325d5a6..294ab1509 100644 --- a/Source/SourceKittenFramework/File.swift +++ b/Source/SourceKittenFramework/File.swift @@ -149,6 +149,50 @@ public final class File { return substring?.removingCommonLeadingWhitespaceFromLines() .trimmingWhitespaceAndOpeningCurlyBrace() } + + /** + Parse the annotated declaration for any usr links. If they are external (i.e. apple ones) include a link to the external documentation. + + - parameter dictionary: SourceKit dictionary to extract declaration from. + + - returns: Source declaration if successfully parsed, with any usr links. Example: `public class Test : `. + */ + public func parseAnnotatedDeclaration(_ dictionary: [String: SourceKitRepresentable]) -> String? { + guard let annotated = SwiftDocKey.getAnnotatedDeclaration(dictionary) else { + return nil + } + + guard let decl = SWXMLHash.parse(annotated).children.first?.element else { + return nil + } + + return recurseAnnotatedXML(decl) + } + + private func recurseAnnotatedXML(_ element: SWXMLHash.XMLElement) -> String { + let parsed = element.children.map({ (child) -> String in + if let text = child as? TextElement { + return text.text + } + + if let xml = child as? SWXMLHash.XMLElement { + let content = recurseAnnotatedXML(xml) + if let usr = xml.attribute(by: "usr") { + if let url = USRResolver.shared.resolveExternalURL(usr: usr.text) { + return "\(content)" + } else { + return "\(content)" + } + } else { + return content + } + } + + return child.description + }) + + return parsed.joined(separator: "") + } /** Parse line numbers containing the declaration's implementation from SourceKit dictionary. @@ -210,6 +254,11 @@ public final class File { if let parsedDeclaration = parseDeclaration(dictionary) { dictionary[SwiftDocKey.parsedDeclaration.rawValue] = parsedDeclaration } + + // Parse annotated declaration and add to dictionary + if let parsedAnnotatedDeclaration = parseAnnotatedDeclaration(dictionary) { + dictionary[SwiftDocKey.parsedAnnotatedDeclaration.rawValue] = parsedAnnotatedDeclaration + } // Parse scope range and add to dictionary if let parsedScopeRange = parseScopeRange(dictionary) { @@ -414,7 +463,6 @@ public final class File { */ internal func addDocComments(dictionary: [String: SourceKitRepresentable], finder: SyntaxMap.DocCommentFinder) -> [String: SourceKitRepresentable] { var dictionary = dictionary - // special-case skip 'enumcase': has same offset as child 'enumelement' if let kind = SwiftDocKey.getKind(dictionary).flatMap(SwiftDeclarationKind.init), kind != .enumcase, @@ -435,6 +483,35 @@ public final class File { return dictionary } + + internal func parseDocComments(dictionary: [String: SourceKitRepresentable], parentUSR: String? = nil) -> [String: SourceKitRepresentable] { + var dictionary = dictionary + let currentUSR = dictionary[SwiftDocKey.usr.rawValue] as? String ?? "" + let kind = SwiftDocKey.getKind(dictionary) + if let docComment = dictionary[SwiftDocKey.documentationComment.rawValue] as? String { + var result = docComment + var start = result.startIndex + while var range = result.range(of: "`.*?`", options: .regularExpression, range: start..\(code)`" + result = result.replacingCharacters(in: range, with: replacement) + range = range.lowerBound.. [String: Any] { set(.name, decl.name) set(.usr, decl.usr) set(.parsedDeclaration, decl.declaration) + set(.parsedAnnotatedDeclaration, decl.annotatedDeclaration) set(.documentationComment, decl.commentBody) set(.parsedScopeStart, Int(decl.extent.start.line)) set(.parsedScopeEnd, Int(decl.extent.end.line)) diff --git a/Source/SourceKittenFramework/Module.swift b/Source/SourceKittenFramework/Module.swift index 65e9d36fc..7df6817b2 100644 --- a/Source/SourceKittenFramework/Module.swift +++ b/Source/SourceKittenFramework/Module.swift @@ -22,7 +22,8 @@ public struct Module { public var docs: [SwiftDocs] { var fileIndex = 1 let sourceFilesCount = sourceFiles.count - return sourceFiles.sorted().compactMap { + // HACKY AF CURRENTLY!! NEED TO ADD THIS SOMEWHERE ELSE; BUT WHERE? + var d : [SwiftDocs] = sourceFiles.sorted().compactMap { let filename = $0.bridge().lastPathComponent if let file = File(path: $0) { fputs("Parsing \(filename) (\(fileIndex)/\(sourceFilesCount))\n", stderr) @@ -32,6 +33,19 @@ public struct Module { fputs("Could not parse `\(filename)`. Please open an issue at https://github.com/jpsim/SourceKitten/issues with the file contents.\n", stderr) return nil } + + var d2 : [SwiftDocs] = [] + + for var sd in d { + USRResolver.shared.register(docs: sd.docsDictionary) + } + + for var sd in d { + sd.parseDocComments() + d2.append(sd) + } + + return d2 } public init?(spmName: String) { diff --git a/Source/SourceKittenFramework/SHA1.swift b/Source/SourceKittenFramework/SHA1.swift new file mode 100644 index 000000000..f7c6ec1d4 --- /dev/null +++ b/Source/SourceKittenFramework/SHA1.swift @@ -0,0 +1,188 @@ +// +// SHA1.swift +// SourceKittenFramework +// +// https://github.com/idrougge/sha1-swift +// +// SHA-1 implementation in Swift 4 +// $AUTHOR: Iggy Drougge +// $VER: 2.3.1 + +import Foundation + +/// Left rotation (or cyclic shift) operator +infix operator <<< : BitwiseShiftPrecedence +private func <<< (lhs:UInt32, rhs:UInt32) -> UInt32 { + return lhs << rhs | lhs >> (32-rhs) +} + +public struct SHA1 { + // One chunk consists of 80 big-endian longwords (32 bits, unsigned) + private static let CHUNKSIZE=80 + // SHA-1 magic words + private static let h0:UInt32 = 0x67452301 + private static let h1:UInt32 = 0xEFCDAB89 + private static let h2:UInt32 = 0x98BADCFE + private static let h3:UInt32 = 0x10325476 + private static let h4:UInt32 = 0xC3D2E1F0 + + /************************************************** + * SHA1.context * + * The context struct contains volatile variables * + * as well as the actual hashing function. * + **************************************************/ + private struct context { + // Initialise variables: + var h:[UInt32]=[SHA1.h0,SHA1.h1,SHA1.h2,SHA1.h3,SHA1.h4] + + // Process one chunk of 80 big-endian longwords + mutating func process(chunk:inout ContiguousArray) { + for i in 0..<16 { + chunk[i] = chunk[i].bigEndian // The numbers must be big-endian + } + //chunk=chunk.map{$0.bigEndian} // The numbers must be big-endian + for i in 16...79 { // Extend the chunk to 80 longwords + chunk[i] = (chunk[i-3] ^ chunk[i-8] ^ chunk[i-14] ^ chunk[i-16]) <<< 1 + } + + // Initialise hash value for this chunk: + var a,b,c,d,e,f,k,temp:UInt32 + a=h[0]; b=h[1]; c=h[2]; d=h[3]; e=h[4] + f=0x0; k=0x0 + + // Main loop + for i in 0...79 { + switch i { + case 0...19: + f = (b & c) | ((~b) & d) + k = 0x5A827999 + case 20...39: + f = b ^ c ^ d + k = 0x6ED9EBA1 + case 40...59: + f = (b & c) | (b & d) | (c & d) + k = 0x8F1BBCDC + case 60...79: + f = b ^ c ^ d + k = 0xCA62C1D6 + default: break + } + temp = a <<< 5 &+ f &+ e &+ k &+ chunk[i] + e = d + d = c + c = b <<< 30 + b = a + a = temp + //print(String(format: "t=%d %08X %08X %08X %08X %08X", i, a, b, c, d, e)) + } + + // Add this chunk's hash to result so far: + h[0] = h[0] &+ a + h[1] = h[1] &+ b + h[2] = h[2] &+ c + h[3] = h[3] &+ d + h[4] = h[4] &+ e + } + } + + /************************************************** + * processData() * + * All inputs are processed as NSData. * + * This function splits the data into chunks of * + * 16 longwords (64 bytes, 512 bits), * + * padding the chunk as necessary. * + **************************************************/ + private static func process(data: inout Data) -> SHA1.context? { + var context=SHA1.context() + var w = ContiguousArray(repeating: 0x00000000, count: CHUNKSIZE) // Initialise empty chunk + let ml=data.count << 3 // Message length in bits + var range = Range(0..<64) // A chunk is 64 bytes + + // If the remainder of the message is more than or equal 64 bytes + while data.count >= range.upperBound { + //print("Reading \(range.count) bytes @ position \(range.lowerBound)") + w.withUnsafeMutableBufferPointer{ dest in + _=data.copyBytes(to: dest, from: range) // Retrieve one chunk + } + context.process(chunk: &w) // Process the chunk + range = Range(range.upperBound..(repeating: 0x00000000, count: CHUNKSIZE) // Initialise empty chunk + range = Range(range.lowerBound.. 56 { + context.process(chunk: &w) + w = ContiguousArray(repeating: 0x00000000, count: CHUNKSIZE) + } + + // The last 64 bits of the last chunk must contain the message length in big-endian format + w[15] = UInt32(ml).bigEndian + context.process(chunk: &w) // Process the last chunk + + // The context (or nil) is returned, containing the hash in the h[] array + return context + } + + /************************************************** + * hexString() * + * Render the hash as a hexadecimal string * + **************************************************/ + private static func hexString(_ context:SHA1.context?) -> String? { + guard let c=context else {return nil} + return String(format: "%08X %08X %08X %08X %08X", c.h[0], c.h[1], c.h[2], c.h[3], c.h[4]) + } + + /************************************************** + * dataFromFile() * + * Fetch the contents of a file as NSData * + * for processing by processData() * + **************************************************/ + private static func dataFromFile(named filename:String) -> SHA1.context? { + guard var file = try? Data(contentsOf: URL(fileURLWithPath: filename)) else {return nil} + return process(data: &file) + } + + /************************************************** + * PUBLIC METHODS * + **************************************************/ + + /// Return a hexadecimal hash from a file + static public func hexString(fromFile filename:String) -> String? { + return hexString(SHA1.dataFromFile(named: filename)) + } + + /// Return the hash of a file as an array of Ints + public static func hash(fromFile filename:String) -> [Int]? { + return dataFromFile(named: filename)?.h.map{Int($0)} + } + + /// Return a hexadecimal hash from NSData + public static func hexString(from data: inout Data) -> String? { + return hexString(SHA1.process(data: &data)) + } + + /// Return the hash of NSData as an array of Ints + public static func hash(from data: inout Data) -> [UInt32]? { + return process(data: &data)?.h + } + + /// Return a hexadecimal hash from a string + public static func hexString(from str:String) -> String? { + guard var data = str.data(using: .utf8) else { return nil } + return hexString(SHA1.process(data: &data)) + } + + /// Return the hash of a string as an array of Ints + public static func hash(from str:String) -> [Int]? { + guard var data = str.data(using: .utf8) else { return nil } + return process(data: &data)?.h.map{Int($0)} + } +} diff --git a/Source/SourceKittenFramework/SourceDeclaration.swift b/Source/SourceKittenFramework/SourceDeclaration.swift index 02fe9497f..e5fff47a2 100644 --- a/Source/SourceKittenFramework/SourceDeclaration.swift +++ b/Source/SourceKittenFramework/SourceDeclaration.swift @@ -38,8 +38,10 @@ public struct SourceDeclaration { public let name: String? public let usr: String? public let declaration: String? + public var annotatedDeclaration: String? public let documentation: Documentation? public let commentBody: String? + public let commentBodyExtent: (start: SourceLocation, end: SourceLocation)? public var children: [SourceDeclaration] public let swiftDeclaration: String? public let swiftName: String? @@ -122,23 +124,69 @@ public struct SourceDeclaration { } extension SourceDeclaration { - init?(cursor: CXCursor, compilerArguments: [String]) { + init?(cursor: CXCursor, compilerArguments: [String], index: CXIndex? = nil) { guard cursor.shouldDocument() else { return nil } + let translation = clang_Cursor_getTranslationUnit(cursor) + type = cursor.objCKind() location = cursor.location() extent = cursor.extent() name = cursor.name() usr = cursor.usr() declaration = cursor.declaration() + annotatedDeclaration = "" documentation = Documentation(comment: cursor.parsedComment()) commentBody = cursor.commentBody() + commentBodyExtent = cursor.commentBodyExtent() children = cursor.compactMap({ SourceDeclaration(cursor: $0, compilerArguments: compilerArguments) }).rejectPropertyMethods() (swiftDeclaration, swiftName) = cursor.swiftDeclarationAndName(compilerArguments: compilerArguments) availability = cursor.platformAvailability() + + let declarationStart = extent.start + var declarationEnd = declarationStart + var references = [(NSRange, String)]() + cursor.tokensAndCursors() { (tokens, cursors) in + for i in 0.. Bool in + decl.extent.start.offset <= tokenExtent.start.offset || (decl.commentBodyExtent?.start.offset ?? 100000) <= tokenExtent.start.offset + }) || value == "@end" { + break + } + + let referencedUSR = clang_getCursorReferenced(tCursor).usr() + if referencedUSR != usr && tCursor.kind.rawValue >= CXCursor_FirstRef.rawValue && tCursor.kind.rawValue <= CXCursor_LastRef.rawValue { + let range = NSMakeRange(Int(tokenExtent.start.offset - declarationStart.offset), Int(tokenExtent.end.offset - tokenExtent.start.offset)) + references.append((range, referencedUSR ?? "")) + } + + declarationEnd = tokenExtent.end + } + } + + annotatedDeclaration = try! String(contentsOfFile: extent.start.file, encoding: .utf8) + annotatedDeclaration = annotatedDeclaration!.substringWithSourceRange(start: declarationStart, end: declarationEnd)! + var rangeOffset = 0 + print(references) + for (range, usr) in references { + let updatedRange = NSMakeRange(range.location + rangeOffset, range.length) + let newRange = Range.init(updatedRange, in: annotatedDeclaration!)! + let value = annotatedDeclaration![newRange] + var replacement = "\(value)" + if let link = USRResolver.shared.resolveExternalURL(usr: usr) { + replacement = "\(value)" + } + rangeOffset += replacement.count - value.count + annotatedDeclaration = annotatedDeclaration!.replacingCharacters(in: newRange, with: replacement) + } } } diff --git a/Source/SourceKittenFramework/String+SourceKitten.swift b/Source/SourceKittenFramework/String+SourceKitten.swift index e81a43083..39dd8968f 100644 --- a/Source/SourceKittenFramework/String+SourceKitten.swift +++ b/Source/SourceKittenFramework/String+SourceKitten.swift @@ -505,7 +505,7 @@ extension String { line: UInt32((self as NSString).lineRangeWithByteRange(start: markByteRange.location, length: 0)!.start), column: 1, offset: UInt32(markByteRange.location)) return SourceDeclaration(type: .mark, location: location, extent: (location, location), name: markString, - usr: nil, declaration: nil, documentation: nil, commentBody: nil, children: [], + usr: nil, declaration: nil, annotatedDeclaration: nil, documentation: nil, commentBody: nil, commentBodyExtent: nil, children: [], swiftDeclaration: nil, swiftName: nil, availability: nil) } } diff --git a/Source/SourceKittenFramework/SwiftDeclarationKind.swift b/Source/SourceKittenFramework/SwiftDeclarationKind.swift index 11455a530..8cf52d792 100644 --- a/Source/SourceKittenFramework/SwiftDeclarationKind.swift +++ b/Source/SourceKittenFramework/SwiftDeclarationKind.swift @@ -8,7 +8,7 @@ /// Swift declaration kinds. /// Found in `strings SourceKitService | grep source.lang.swift.decl.`. -public enum SwiftDeclarationKind: String { +public enum SwiftDeclarationKind: String, Codable { /// `associatedtype`. case `associatedtype` = "source.lang.swift.decl.associatedtype" /// `class`. diff --git a/Source/SourceKittenFramework/SwiftDocKey.swift b/Source/SourceKittenFramework/SwiftDocKey.swift index b8aa878ad..7a7c1d814 100644 --- a/Source/SourceKittenFramework/SwiftDocKey.swift +++ b/Source/SourceKittenFramework/SwiftDocKey.swift @@ -53,6 +53,8 @@ public enum SwiftDocKey: String { case docColumn = "key.doc.column" /// Documentation comment (String). case documentationComment = "key.doc.comment" + /// Parsed Documentation comment (String). + case parsedDocumentationComment = "key.doc.parsed_comment" /// Declaration of documented token (String). case docDeclaration = "key.doc.declaration" /// Discussion documentation of documented token ([SourceKitRepresentable]). @@ -73,6 +75,8 @@ public enum SwiftDocKey: String { case usr = "key.usr" /// Result discussion documentation of documented token ([SourceKitRepresentable]). case parsedDeclaration = "key.parsed_declaration" + /// Parsed declaration annotated with links to any found usr (String). + case parsedAnnotatedDeclaration = "key.parsed_annotated_decl" /// Type of documented token (String). case parsedScopeEnd = "key.parsed_scope.end" /// USR of documented token (String). diff --git a/Source/SourceKittenFramework/SwiftDocs.swift b/Source/SourceKittenFramework/SwiftDocs.swift index fbc736c14..438a95171 100644 --- a/Source/SourceKittenFramework/SwiftDocs.swift +++ b/Source/SourceKittenFramework/SwiftDocs.swift @@ -17,8 +17,8 @@ public struct SwiftDocs { public let file: File /// Docs information as an [String: SourceKitRepresentable]. - public let docsDictionary: [String: SourceKitRepresentable] - + public var docsDictionary: [String: SourceKitRepresentable] + /** Create docs for the specified Swift file and compiler arguments. @@ -30,7 +30,8 @@ public struct SwiftDocs { self.init( file: file, dictionary: try Request.editorOpen(file: file).send(), - cursorInfoRequest: Request.cursorInfoRequest(filePath: file.path, arguments: arguments) + cursorInfoRequest: Request.cursorInfoRequest(filePath: file.path, arguments: arguments), + arguments: arguments ) } catch let error as Request.Error { fputs(error.description, stderr) @@ -47,7 +48,7 @@ public struct SwiftDocs { - parameter dictionary: editor.open response from SourceKit. - parameter cursorInfoRequest: SourceKit dictionary to use to send cursorinfo request. */ - public init(file: File, dictionary: [String: SourceKitRepresentable], cursorInfoRequest: SourceKitObject?) { + public init(file: File, dictionary: [String: SourceKitRepresentable], cursorInfoRequest: SourceKitObject?, arguments: [String]) { self.file = file var dictionary = dictionary let syntaxMapData = dictionary.removeValue(forKey: SwiftDocKey.syntaxMap.rawValue) as! [SourceKitRepresentable] @@ -64,6 +65,10 @@ public struct SwiftDocs { } docsDictionary = file.addDocComments(dictionary: dictionary, syntaxMap: syntaxMap) } + + public mutating func parseDocComments() { + self.docsDictionary = self.file.parseDocComments(dictionary: self.docsDictionary) + } } // MARK: CustomStringConvertible diff --git a/Source/SourceKittenFramework/USRResolver.swift b/Source/SourceKittenFramework/USRResolver.swift new file mode 100644 index 000000000..aa304ffc9 --- /dev/null +++ b/Source/SourceKittenFramework/USRResolver.swift @@ -0,0 +1,387 @@ +// +// USRResolver.swift +// SourceKittenFramework +// +// Created by Leonardo Galli on 17.06.18. +// + +import Foundation +import CSQLite + +class USRResolver { + public static let shared = USRResolver() + + private let searchPaths : [String] + + private var db: OpaquePointer? = nil + + private let documentationURL = "https://developer.apple.com/documentation/" + + private var usrCache : [String : String] = [:] + + private var codeUsrCache : [String : String] = [:] + + private var index: [String: NameEntity] = [:] + + private init() { + self.searchPaths = [ + xcodeDefaultToolchainOverride, + toolchainDir, + xcrunFindPath, + /* + These search paths are used when `xcode-select -p` points to + "Command Line Tools OS X for Xcode", but Xcode.app exists. + */ + applicationsDir?.xcodeDeveloperDir.toolchainDir, + applicationsDir?.xcodeBetaDeveloperDir.toolchainDir, + userApplicationsDir?.xcodeDeveloperDir.toolchainDir, + userApplicationsDir?.xcodeBetaDeveloperDir.toolchainDir + ].compactMap { path in + if let fullPath = path?.deleting(lastPathComponents: 3).appending(pathComponent: "SharedFrameworks/DNTDocumentationSupport.framework/Resources/external/map.db"), fullPath.isFile { + return fullPath + } + return nil + } + + self.loadDatabase() + } + + deinit { + sqlite3_close(db) + } + + private func loadDatabase() { + guard let path = self.searchPaths.first else { return } + //print("Loading map.db from \(path)") + if sqlite3_open(path, &db) == SQLITE_OK { + //print("Successfully opened connection to database at \(path)") + } else { + //print("Unable to open documentation database.") + } + } + + /// Find a usr that best matches a given code snippet and the current context. + /// The code snipped and either be full on compilable swift code (`compilerArgs` are required) + /// or just a representation of the hirarchy, called "dot notation" (e.g. `Class.function`). + /// For a more indepth overview of "dot notation", see `findUsingDotNotation`. + /// + /// - Parameters: + /// - code: Either a swift code snippet or "dot notation". + /// - context: The context where the code snippet was mentioned. Since this is usually a doc comment, it will include the usr of the doc comment location, the parent usr and all children. + /// - compilerArgs: The args for compiling the code snippet (only useful if it's actual swift code). + /// - Returns: Returns a usr when found. + public func resolveUSR(code: String, context: NameEntity, compilerArgs: [String]? = nil) -> String? { + let cacheKey = code + context.usr + (context.parentUSR ?? "") + if let cached = self.codeUsrCache[cacheKey] { + return cached + } + + if let compilerArgs = compilerArgs { + if let usr = self.findUsingCursorInfo(code: code, compilerArgs: compilerArgs) { + self.codeUsrCache[cacheKey] = usr + return usr + } + } + + if let usr = self.findUsingDotNotation(code: code, context: context) { + self.codeUsrCache[cacheKey] = usr + return usr + } + + return nil + } + + public func findUsingCursorInfo(code: String, compilerArgs: [String]) -> String? { + var compilerArgs = compilerArgs + let tempPath = NSTemporaryDirectory().appending(pathComponent: "temp.swift") + if tempPath.isFile { + try? FileManager.default.removeItem(atPath: tempPath) + } + try? code.write(toFile: tempPath, atomically: true, encoding: String.Encoding.utf8) + compilerArgs.append(tempPath) + let requestObj = ["key.request": UID("source.request.cursorinfo"), "key.compilerargs" : compilerArgs, "key.sourcefile": tempPath, "key.offset": code.lengthOfBytes(using: .utf8) - 2] as SourceKitObject + let request = Request.customRequest(request: requestObj) + + do { + let response = try request.send() + return response[SwiftDocKey.usr.rawValue] as? String + } catch { + print("Error: \(error)") + } + + return nil + } + + public func findUsingDotNotation(code: String, context: NameEntity) -> String? { + let parts = code.split(regex: "(? 0 { + if entities.count == 1 { + return entities.first?.usr + } + + // first let's filter by size and take the smallest. + let withSize = entities.map { (ent) -> (NameEntity, Int) in + return (ent, ent.name.count) + }.sorted { (a, b) -> Bool in + a.1 < b.1 + } + + let smallest = withSize.first! + let allSmall = withSize.filter { (ent) -> Bool in + ent.1 == smallest.1 + } + + if allSmall.count == 1 { + return smallest.0.usr + } + + //If we have multiple entities with the same name, we need to look at the context! + let withScore = allSmall.map { (arg) -> (NameEntity, Int) in + let (ent, _) = arg + //We swap the entity and context and score them again, since context could also be a child of the entity. + return (ent, score(ent: ent, context: context) + score(ent: context, context: ent)) + }.sorted { (a, b) -> Bool in + a.1 > b.1 + } + let highest = withScore.first! + let allHighest = withScore.filter { (ent) -> Bool in + ent.1 == highest.1 + } + + if allHighest.count > 1 { + let joined = allHighest.map { (ent) -> String in + return ent.0.name + }.joined(separator: ", ") + //print("WARNING: Could not uniquely resolve \(code). Found posibilities: \(joined).", stderr) + } + + return allHighest.first?.0.usr + } + + return nil + } + + /// Creates a score indicating how likely it is, that we have the correct match for a given context. + /// See `resolveUSR(...)` for what is meant by context. + /// + /// - Parameters: + /// - ent: The entity to score. + /// - context: See `resolveUSR(...)` for what is meant by context. + /// - Returns: A score indicating how likely the given entity `ent` is to be the one we want inside `context`. + private func score(ent: NameEntity, context: NameEntity) -> Int { + var score = 0 + //If the parent is the same, this is very likely to be correct. However, could still be off, so we use more mesures + if ent.parentUSR == context.parentUSR { + score += 100 + } + //We might be on a doc comment on a parent (e.g. class doc comment), so the parent is actually the current context. + if ent.parentUSR == context.usr { + score += 75 + } + + for child in ent.children { + // We increase the score for any same children, as that further increases the chances of the correct match. + if context.children.contains(child) { + score += 10 + } + //The entity we found, might be a parent of a parent. + if context.parentUSR == child { + score += 25 + } + } + + return score + } + + internal func findUsingDotNotation(parts: [String], index: [NameEntity]) -> [NameEntity] { + var current = parts.first + current = NSRegularExpression.escapedPattern(for: current ?? "") + current = current?.replacingOccurrences(of: "\\.\\.\\.", with: "[^)]*") + current = current?.replacingOccurrences(of: "_:", with: "[^:]*:") + //print(current) + let nextParts = parts.dropFirst() + let entities = index.filter { (ent) -> Bool in + return ent.name.range(of: current ?? "", options: .regularExpression, range: nil, locale: nil) != nil + } + + if nextParts.count == 0 { + if entities.count > 1 { + //print("WARNING: Found multiple entities for \(current).") + } + return entities + } + + if entities.count > 0 { + let children = entities.flatMap { (ent) -> [NameEntity] in + ent.children.map({ (usr) -> NameEntity in + return self.index[usr]! + }) + } + + return self.findUsingDotNotation(parts: Array(nextParts), index: children) + } + + return [] + } + + public func register(docs: [String: SourceKitRepresentable], parentUSR : String? = nil) { + var children : [String] = [] + // This is the usr of the current structure, being the parent of it's substructures. + let parent = docs[SwiftDocKey.usr.rawValue] as? String + if let substructures = SwiftDocKey.getSubstructure(docs) { + for substructure in substructures { + self.register(docs: substructure, parentUSR: parent) + } + + children = self.getChildUSRs(substructures: substructures) + } + + if let name = SwiftDocKey.getName(docs) { + //This is actually an entity and not something top level! + guard let kind = SwiftDeclarationKind(rawValue: SwiftDocKey.getKind(docs) ?? ""), let usr = docs[SwiftDocKey.usr.rawValue] as? String else { + return + } + + let entity = NameEntity(usr: usr, name: name, children: children, parentUSR: parentUSR, kind: kind) + self.index[usr] = entity + } + } + + internal func getChildUSRs(substructures: [[String: SourceKitRepresentable]]) -> [String] { + var children : [String] = [] + // We save the usrs of all substructures, so we can have a linked list. + for substructure in substructures { + if let usr = substructure[SwiftDocKey.usr.rawValue] as? String { + children.append(usr) + } else { + // If the child does not have a usr, it might be nested, e.g. like an enum case statement. + if let subsub = SwiftDocKey.getSubstructure(substructure) { + children.append(contentsOf: self.getChildUSRs(substructures: subsub)) + } + } + } + + return children + } + + public func resolveExternalURL(usr: String, language: DocumentationSourceLanguage = DocumentationSourceLanguage.swift) -> String? { + if let cached = self.usrCache[usr] { + return cached + } + + if let url = self.findInAppleDocs(usr: usr, language: language) { + self.usrCache[usr] = url + return url + } + + return nil + } + + public func findInAppleDocs(usr: String, language: DocumentationSourceLanguage = DocumentationSourceLanguage.swift) -> String? { + guard var data = usr.data(using: .utf8), let hashBytes = SHA1.hash(from: &data) else { return nil } + var hashData = Data.init(count: hashBytes.count*4) + for (index, byte) in hashBytes.enumerated() { + let bigEndian = byte.bigEndian + for i in (0..<4) { + hashData[index*4 + i] = UInt8(truncatingIfNeeded: bigEndian >> UInt8(i*8)) + } + } + //let hashData = hashBytes.withUnsafeBytes { Data(buffer: ($0.bindMemory(to: Int.self))) } + let hash = hashData.base64EncodedString() + + // Foundation needs to be special so it has different base64 chars. + let correctedHash = hash.replacingOccurrences(of: "+", with: "-").replacingOccurrences(of: "/", with: "_") + + // This part is stored in the map db. + let uuid = correctedHash[correctedHash.startIndex.. String? { + guard let db = self.db else { return nil } + + var referencePath : String? = nil + var queryStatement: OpaquePointer? = nil + + if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { + + if sqlite3_step(queryStatement) == SQLITE_ROW { + let referencePathCol = sqlite3_column_text(queryStatement, 4) + referencePath = String(cString: referencePathCol!) + } else { + //print("Query returned no results") + } + } else { + //print("SELECT statement could not be prepared") + } + + sqlite3_finalize(queryStatement) + + return referencePath + } +} + +public enum DocumentationSourceLanguage: Int { + case swift + case objc + case javascript + + public var character: String { + switch self { + case .swift: + return "s" + case .objc: + return "c" + case .javascript: + return "j" + } + } +} + +public struct NameEntity: Codable { + public let usr: String + + public let name: String + + public var children: [String] = [] + + public var parentUSR: String? = nil + + public var kind: SwiftDeclarationKind +} + +extension String { + func ranges(of string: String, options: CompareOptions = .literal) -> [Range] { + var result: [Range] = [] + var start = startIndex + while let range = range(of: string, options: options, range: start.. [Range] { + var result: [Range] = [] + var start = startIndex + while let range = range(of: string, options: options, range: start.. [String] { + return self.ranges(between: regex, options: .regularExpression).map { String(self[$0]) } + } +} diff --git a/Source/SourceKittenFramework/library_wrapper.swift b/Source/SourceKittenFramework/library_wrapper.swift index 42b49c512..15b4650fe 100644 --- a/Source/SourceKittenFramework/library_wrapper.swift +++ b/Source/SourceKittenFramework/library_wrapper.swift @@ -140,18 +140,18 @@ internal let linuxDefaultLibPath = "/usr/lib" /// /// `launch-with-toolchain` sets the toolchain path to the /// "XCODE_DEFAULT_TOOLCHAIN_OVERRIDE" environment variable. -private let xcodeDefaultToolchainOverride = env("XCODE_DEFAULT_TOOLCHAIN_OVERRIDE") +internal let xcodeDefaultToolchainOverride = env("XCODE_DEFAULT_TOOLCHAIN_OVERRIDE") /// Returns "TOOLCHAIN_DIR" environment variable /// /// `Xcode`/`xcodebuild` sets the toolchain path to the /// "TOOLCHAIN_DIR" environment variable. -private let toolchainDir = env("TOOLCHAIN_DIR") +internal let toolchainDir = env("TOOLCHAIN_DIR") /// Returns toolchain directory that parsed from result of `xcrun -find swift` /// /// This is affected by "DEVELOPER_DIR", "TOOLCHAINS" environment variables. -private let xcrunFindPath: String? = { +internal let xcrunFindPath: String? = { let pathOfXcrun = "/usr/bin/xcrun" if !FileManager.default.isExecutableFile(atPath: pathOfXcrun) { @@ -188,13 +188,13 @@ private let xcrunFindPath: String? = { return xcrunFindPath }() -private let applicationsDir: String? = +internal let applicationsDir: String? = NSSearchPathForDirectoriesInDomains(.applicationDirectory, .systemDomainMask, true).first -private let userApplicationsDir: String? = +internal let userApplicationsDir: String? = NSSearchPathForDirectoriesInDomains(.applicationDirectory, .userDomainMask, true).first -private extension String { +internal extension String { var toolchainDir: String { return appending(pathComponent: "Toolchains/XcodeDefault.xctoolchain") } diff --git a/sourcekitten.xcodeproj/xcshareddata/xcschemes/sourcekitten.xcscheme b/sourcekitten.xcodeproj/xcshareddata/xcschemes/sourcekitten.xcscheme index 872d89f5e..01fb52519 100644 --- a/sourcekitten.xcodeproj/xcshareddata/xcschemes/sourcekitten.xcscheme +++ b/sourcekitten.xcodeproj/xcshareddata/xcschemes/sourcekitten.xcscheme @@ -17,7 +17,7 @@ BlueprintIdentifier = "D0E7B63119E9C64500EDBA4D" BuildableName = "sourcekitten.app" BlueprintName = "sourcekitten" - ReferencedContainer = "container:SourceKitten.xcodeproj"> + ReferencedContainer = "container:sourcekitten.xcodeproj"> @@ -35,7 +35,7 @@ BlueprintIdentifier = "D0D1217619E87B05005E4BAA" BuildableName = "SourceKittenFrameworkTests.xctest" BlueprintName = "SourceKittenFrameworkTests" - ReferencedContainer = "container:SourceKitten.xcodeproj"> + ReferencedContainer = "container:sourcekitten.xcodeproj"> @@ -45,7 +45,7 @@ BlueprintIdentifier = "D0E7B63119E9C64500EDBA4D" BuildableName = "sourcekitten.app" BlueprintName = "sourcekitten" - ReferencedContainer = "container:SourceKitten.xcodeproj"> + ReferencedContainer = "container:sourcekitten.xcodeproj"> @@ -70,7 +70,7 @@ BlueprintIdentifier = "D0E7B63119E9C64500EDBA4D" BuildableName = "sourcekitten.app" BlueprintName = "sourcekitten" - ReferencedContainer = "container:SourceKitten.xcodeproj"> + ReferencedContainer = "container:sourcekitten.xcodeproj"> @@ -101,7 +101,7 @@ BlueprintIdentifier = "D0E7B63119E9C64500EDBA4D" BuildableName = "sourcekitten.app" BlueprintName = "sourcekitten" - ReferencedContainer = "container:SourceKitten.xcodeproj"> + ReferencedContainer = "container:sourcekitten.xcodeproj">