Skip to content
86 changes: 59 additions & 27 deletions apps/swift-ios/App/NativeFeatureClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ private struct T3ConnectManagedCleanupError: LocalizedError {
@MainActor
final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
FeatureProjectCreationClient, FeatureWorkspaceAssetResolving, FeatureAttachmentAssetResolving,
FeatureFeedbackSubmitting, T3ConnectCapable
FeatureFeedbackSubmitting, FeatureContextAttachmentResolving, T3ConnectCapable
{
private static let maximumRetainedThreadDetails = 6
private static let t3ConnectLogger = Logger(
Expand Down Expand Up @@ -1529,7 +1529,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
shell: shellsByEnvironmentID[environment.id]
)
let title = Self.title(from: prompt, hasAttachments: !attachments.isEmpty)
let uploads = try makeUploadAttachments(attachments)
let uploads = try await makeUploadAttachments(attachments)
if !uploads.isEmpty { _ = try await client.serverConfig() }
let runtime = coreRuntimeMode(runtimeMode)
let interaction = coreInteractionMode(interactionMode)
Expand Down Expand Up @@ -2208,7 +2208,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
throw NativeFeatureClientError.threadNotFound
}
let model = selection.map(coreModelSelection)
let uploads = try makeUploadAttachments(attachments)
let uploads = try await makeUploadAttachments(attachments)
if !uploads.isEmpty { _ = try await client.serverConfig() }
let runtimeMode = coreRuntimeMode(
requestedRuntimeMode ?? mapRuntimeMode(shellThread.runtimeMode)
Expand Down Expand Up @@ -2474,11 +2474,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
throw NativeFeatureClientError.inputRequestNotFound
}
let route = try threadRoute(for: request.threadID)
var uploads: [String: [UploadChatAttachment]] = [:]
for (questionID, attachments) in attachmentsByQuestionID {
uploads[questionID] = try await makeUploadAttachments(attachments)
}
_ = try await route.client.respondToUserInput(
threadID: route.wireID,
requestID: request.wireID,
answers: answers.mapValues(\.jsonValue),
attachmentsByQuestionID: try attachmentsByQuestionID.mapValues(makeUploadAttachments)
attachmentsByQuestionID: uploads
)
inputRoutes[id] = nil
removeCachedInput(id: id, threadID: route.uiID)
Expand Down Expand Up @@ -3433,9 +3437,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
) async throws -> FeatureUploadedAttachmentReference? {
let client = try await projectCreationClient(environmentID: environmentID)
_ = try await client.serverConfig()
let prepared = try await client.prepareAttachment(
makeUploadAttachments([attachment])[0]
)
let uploads = try await makeUploadAttachments([attachment])
let prepared = try await client.prepareAttachment(uploads[0])
return prepared.map {
FeatureUploadedAttachmentReference(
environmentID: $0.environmentID,
Expand Down Expand Up @@ -7097,6 +7100,25 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
return resolved.url
}

func contextAttachmentAssetURL(
environmentID: String, attachment: ComposerContextRecord.Attachment
) async throws -> URL {
try Task.checkCancellation()
guard try await runtime.environments().contains(where: { $0.id == environmentID && $0.isEnabled }) else {
throw ComposerContextClipboardError.sourceUnavailable
}
let client = try await projectCreationClient(environmentID: environmentID)
let generation = environmentGeneration
let resolved = try await client.resolvedAsset(resource: .attachment(
id: attachment.attachmentId, fileName: attachment.name, mimeType: attachment.mimeType
))
try Task.checkCancellation()
guard isKnownClient(client, environmentID: environmentID, generation: generation) else {
throw CancellationError()
}
return resolved.url
}

private func lastActivityDate(
latestUserMessageAt: String?,
latestTurn: OrchestrationLatestTurn?
Expand Down Expand Up @@ -7161,37 +7183,47 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,

private func makeUploadAttachments(
_ attachments: [FeatureUploadAttachment]
) throws -> [UploadChatAttachment] {
) async throws -> [UploadChatAttachment] {
guard attachments.count <= 8 else {
throw NativeFeatureClientError.tooManyAttachments
}
return try attachments.map {
let reference = $0.uploadedReference.map {
UploadedAttachmentReference(
environmentID: $0.environmentID,
attachmentID: $0.attachmentID
)
}
if let ownedFile = $0.ownedFile {
let uploads = try await Task.detached(priority: .userInitiated) {
try attachments.map {
let reference = $0.uploadedReference.map {
UploadedAttachmentReference(
environmentID: $0.environmentID,
attachmentID: $0.attachmentID
)
}
if let ownedFile = $0.ownedFile {
if $0.mimeType.trimmingCharacters(in: .whitespacesAndNewlines).lowercased().hasPrefix("image/") {
return try UploadChatAttachment(
id: $0.id, data: Data(contentsOf: ownedFile.url),
name: $0.name, mimeType: $0.mimeType, uploadedReference: reference
)
}
return try UploadChatAttachment(
id: $0.id,
fileURL: ownedFile.url,
name: $0.name,
mimeType: $0.mimeType,
sizeBytes: ownedFile.byteCount,
uploadedReference: reference,
contextSource: $0.source
)
}
return try UploadChatAttachment(
id: $0.id,
fileURL: ownedFile.url,
data: $0.data,
name: $0.name,
mimeType: $0.mimeType,
sizeBytes: ownedFile.byteCount,
uploadedReference: reference,
contextSource: $0.source
)
}
return try UploadChatAttachment(
id: $0.id,
data: $0.data,
name: $0.name,
mimeType: $0.mimeType,
uploadedReference: reference,
contextSource: $0.source
)
}
}.value
try Task.checkCancellation()
return uploads
}

private func requireScope(_ scope: String, client: T3Client) async throws {
Expand Down
4 changes: 2 additions & 2 deletions apps/swift-ios/Core/ComposerContext.swift
Comment thread
t3dotgg marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import Foundation
public struct ComposerContextRecord: Codable, Equatable, Hashable, Sendable, Identifiable {
public var id: String { contextId }
public let version: Int
public let contextId: String
public var contextId: String
public let label: String
public var payload: Payload

Expand Down Expand Up @@ -103,7 +103,7 @@ public struct ComposerContextRecord: Codable, Equatable, Hashable, Sendable, Ide
public let regionCount: Int?
public let strokeCount: Int?
public let styleChangeDetails: [StyleChange]?
public let screenshotContextId: String?
public var screenshotContextId: String?

private enum CodingKeys: String, CodingKey {
case annotationId, pageUrl, pageTitle, comment, targetSummary, styleChanges
Expand Down
154 changes: 154 additions & 0 deletions apps/swift-ios/Core/ComposerContextClipboard.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import Foundation

public struct ComposerContextClipboardFragment: Codable, Equatable, Sendable {
public struct Source: Codable, Equatable, Sendable {
public let environmentId: String
public var threadId: String?
public var messageId: String?

public init(environmentId: String, threadId: String? = nil, messageId: String? = nil) {
self.environmentId = environmentId
self.threadId = threadId
self.messageId = messageId
}
}

public let version: Int
public let source: Source
public let records: [ComposerContextRecord]

public init(source: Source, records: [ComposerContextRecord]) {
version = 1
self.source = source
self.records = records
}
}

public enum ComposerContextClipboardError: LocalizedError, Equatable {
case invalidFragment
case tooLarge
case missingRecord
case contextLimit
case sourceUnavailable
case draftChanged

public var errorDescription: String? {
switch self {
case .invalidFragment: "The clipboard context is invalid. Copy it again from the source."
case .tooLarge: "This context is too large to copy. Select fewer items."
case .missingRecord: "A copied context item is missing its saved data. Copy it again from the source."
case .contextLimit: "A message can have up to 200 context items. Remove some items and paste again."
case .sourceUnavailable: "Reconnect to the source environment to paste its attachments."
case .draftChanged: "The draft changed while the paste was loading. Paste again."
}
}
}

/// Matches the web/native clipboard contract. Binary data and resolved asset URLs stay out of this payload.
public enum ComposerContextClipboard {
public static let mimeType = "web application/x-t3-context-fragment+json"
public static let maximumCharacters = 16_000_000
public static let maximumRecords = 200

public struct Content: Equatable, Sendable {
public let text: String
public let fragment: ComposerContextClipboardFragment
}

public static func encode(_ fragment: ComposerContextClipboardFragment) throws -> String {
try validate(fragment)
let data = try JSONEncoder.t3.encode(fragment)
let text = String(decoding: data, as: UTF8.self)
guard text.utf16.count <= maximumCharacters else { throw ComposerContextClipboardError.tooLarge }
return text
}

public static func decode(_ text: String) throws -> ComposerContextClipboardFragment {
guard text.utf16.count <= maximumCharacters else { throw ComposerContextClipboardError.tooLarge }
let fragment: ComposerContextClipboardFragment
do {
// Decode every record. An invalid record must not turn into a broken pasted link.
fragment = try JSONDecoder.t3.decode(ComposerContextClipboardFragment.self, from: Data(text.utf8))
} catch {
throw ComposerContextClipboardError.invalidFragment
}
try validate(fragment)
return fragment
}

private static func validate(_ fragment: ComposerContextClipboardFragment) throws {
guard fragment.version == 1, !fragment.source.environmentId.isEmpty,
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Set(fragment.records.map(\.contextId)).count == fragment.records.count else {
throw ComposerContextClipboardError.invalidFragment
}
guard fragment.records.count <= maximumRecords else { throw ComposerContextClipboardError.contextLimit }
guard fragment.records.allSatisfy({ record in
record.label.utf16.count <= 200
&& ComposerContextReferences.parseHref("t3-context://v1/\(record.kind)/\(record.contextId)") != nil
}) else { throw ComposerContextClipboardError.invalidFragment }
}

public static func html(text: String, fragment: String) -> String {
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~")
let encoded = fragment.addingPercentEncoding(withAllowedCharacters: allowed) ?? ""
let escaped = text.replacingOccurrences(of: "&", with: "&amp;")
.replacingOccurrences(of: "<", with: "&lt;").replacingOccurrences(of: ">", with: "&gt;")
return "<pre data-t3-context-fragment=\"\(encoded)\">\(escaped)</pre>"
}

public static func decodeHTML(_ html: String) throws -> ComposerContextClipboardFragment? {
guard html.utf16.count <= maximumCharacters * 9 + 4_096 else {
throw ComposerContextClipboardError.tooLarge
}
let expression = try NSRegularExpression(pattern: #"data-t3-context-fragment=["']([^"']+)["']"#)
guard let match = expression.firstMatch(in: html, range: NSRange(location: 0, length: html.utf16.count)) else {
return nil
}
guard let decoded = (html as NSString).substring(with: match.range(at: 1)).removingPercentEncoding else {
throw ComposerContextClipboardError.invalidFragment
}
return try decode(decoded)
}

/// Include only selected records and the screenshots required by selected annotations.
public static func selected(text: String, fragment: ComposerContextClipboardFragment) throws -> Content {
try validate(fragment)
let references = ComposerContextReferences.collect(text)
let byID = Dictionary(uniqueKeysWithValues: fragment.records.map { ($0.contextId, $0) })
var ids = Set<String>()
for reference in references {
guard let record = byID[reference.contextId], record.kind == reference.kind else {
throw ComposerContextClipboardError.missingRecord
}
ids.insert(record.contextId)
if case let .previewAnnotation(annotation) = record.payload, let screenshot = annotation.screenshotContextId {
guard byID[screenshot]?.kind == "image" else { throw ComposerContextClipboardError.missingRecord }
ids.insert(screenshot)
}
}
return Content(text: text, fragment: .init(source: fragment.source, records: fragment.records.filter { ids.contains($0.contextId) }))
}

public static func reidentified(_ content: Content, createID: () -> String = { UUID().uuidString }) throws -> Content {
let selected = try selected(text: content.text, fragment: content.fragment)
let ids = Dictionary(uniqueKeysWithValues: selected.fragment.records.map { ($0.contextId, createID()) })
guard Set(ids.values).count == ids.count else { throw ComposerContextClipboardError.invalidFragment }
let text = ComposerContextReferences.replace(selected.text) { reference in
let original = (selected.text as NSString).substring(with: reference.range)
return original.replacingOccurrences(
of: "t3-context://v1/\(reference.kind)/\(reference.contextId)",
with: "t3-context://v1/\(reference.kind)/\(ids[reference.contextId]!)"
)
}
let records = selected.fragment.records.map { record in
var result = record
result.contextId = ids[record.contextId]!
if case var .previewAnnotation(annotation) = record.payload, let screenshot = annotation.screenshotContextId {
annotation.screenshotContextId = ids[screenshot]!
result.payload = .previewAnnotation(annotation)
}
return result
}
return Content(text: text, fragment: .init(source: selected.fragment.source, records: records))
}
}
29 changes: 29 additions & 0 deletions apps/swift-ios/Features/Chat/FeatureComposerTextInput.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ struct FeatureComposerTextInput: UIViewRepresentable {
var onPasteTextAttachment: ((String, @escaping @MainActor () -> Bool) -> Void)? = nil
var onPasteTextError: ((String) -> Void)? = nil
var draftOwnerID: String = ""
var onCopyContext: ((String) throws -> Bool)? = nil
var onPasteContext: ((ComposerContextClipboard.Content, String, NSRange) -> Void)? = nil

func makeCoordinator() -> Coordinator {
Coordinator(self)
Expand All @@ -42,6 +44,11 @@ struct FeatureComposerTextInput: UIViewRepresentable {
textView.onPasteTextAttachment = onPasteTextAttachment
textView.draftOwnerID = draftOwnerID
textView.onPasteTextError = onPasteTextError
textView.onCopySelection = { selected in
try onCopyContext?(FeatureInlineSkillProjection.plainText(from: selected)) ?? false
}
textView.onCopyError = onPasteTextError
textView.onPasteContext = onPasteContext
if onDismissKeyboard != nil {
textView.installDismissPanRecognizer()
}
Expand Down Expand Up @@ -76,6 +83,11 @@ struct FeatureComposerTextInput: UIViewRepresentable {
textView.onPasteTextAttachment = onPasteTextAttachment
textView.draftOwnerID = draftOwnerID
textView.onPasteTextError = onPasteTextError
textView.onCopySelection = { selected in
try onCopyContext?(FeatureInlineSkillProjection.plainText(from: selected)) ?? false
}
textView.onCopyError = onPasteTextError
textView.onPasteContext = onPasteContext
textView.isReadOnly = isReadOnly

let previousAttributedText = textView.attributedText ?? NSAttributedString()
Expand Down Expand Up @@ -488,6 +500,7 @@ final class FeatureComposerUITextView: FeatureInlineSkillTextView {
var draftOwnerID = ""
private var pastedTextRequestID = UUID()
var onPasteTextError: ((String) -> Void)?
var onPasteContext: ((ComposerContextClipboard.Content, String, NSRange) -> Void)?
private var wantsFirstResponderOnAttach = false

override var keyCommands: [UIKeyCommand]? {
Expand Down Expand Up @@ -625,6 +638,22 @@ final class FeatureComposerUITextView: FeatureInlineSkillTextView {
// attached screenshot reads as a bug.
override func paste(_ sender: Any?) {
guard !isReadOnly else { return }
do {
if let content = try FeatureContextClipboard.read() {
guard let onPasteContext, markedTextRange == nil else {
throw ComposerContextClipboardError.draftChanged
}
onPasteContext(
content,
FeatureInlineSkillProjection.plainText(from: attributedText),
FeatureInlineSkillProjection.plainRange(for: selectedRange, in: attributedText)
)
return
}
} catch {
onPasteTextError?(error.localizedDescription)
return
}
if acceptsImages {
let imageProviders = UIPasteboard.general.itemProviders.filter {
$0.hasItemConformingToTypeIdentifier(UTType.image.identifier)
Expand Down
Loading
Loading