diff --git a/builder.go b/builder.go index ba2d743e..7aebb9d0 100644 --- a/builder.go +++ b/builder.go @@ -41,6 +41,7 @@ type serverBuilder struct { disableParallelism bool imapLimits limits.IMAP disableIMAPAuthenticate bool + enableGmailExtension bool uidValidityGenerator imap.UIDValidityGenerator panicHandler async.PanicHandler dbCI db.ClientInterface @@ -138,6 +139,7 @@ func (builder *serverBuilder) build() (*Server, error) { reporter: builder.reporter, disableParallelism: builder.disableParallelism, disableIMAPAuthenticate: builder.disableIMAPAuthenticate, + enableGmailExtension: builder.enableGmailExtension, uidValidityGenerator: builder.uidValidityGenerator, panicHandler: builder.panicHandler, observabilitySender: builder.observabilitySender, diff --git a/connector/connector.go b/connector/connector.go index 2b3f1092..a0220f74 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -57,6 +57,19 @@ type Connector interface { // MarkMessagesForwarded sets the forwarded value of the give messages. MarkMessagesForwarded(ctx context.Context, cache IMAPStateWrite, messageIDs []imap.MessageID, forwarded bool) error + // MarkMessagesWithGmailLabels applies or removes Gmail-style labels (X-GM-EXT-1 extension). + // Labels are identified by name. If add is true, labels are applied; if false, they are removed. + // This must NOT modify folder membership — messages stay in their current mailbox (e.g., INBOX). + MarkMessagesWithGmailLabels(ctx context.Context, cache IMAPStateWrite, messageIDs []imap.MessageID, labels []string, add bool) error + + // GetGmailLabels retrieves the Gmail-style label names for the given message. + // Note: this can get called from different go routines. + GetGmailLabels(ctx context.Context, messageID imap.MessageID) ([]string, error) + + // GetGmailLabelMailboxID returns the IMAP mailbox ID for a given Gmail label name. + // Used for efficient SEARCH X-GM-LABELS operations via local DB lookups. + GetGmailLabelMailboxID(ctx context.Context, label string) (imap.MailboxID, bool) + // GetUpdates returns a stream of updates that the gluon server should apply. GetUpdates() <-chan imap.Update diff --git a/connector/dummy.go b/connector/dummy.go index 04ae030a..beba0f06 100644 --- a/connector/dummy.go +++ b/connector/dummy.go @@ -276,6 +276,59 @@ func (conn *Dummy) MarkMessagesForwarded(ctx context.Context, cache IMAPStateWri return nil } +// MarkMessagesWithGmailLabels applies or removes Gmail-style labels. Each label +// is backed by a non-exclusive mailbox; (un)labelling only (un)links the message +// from that mailbox and never touches folder membership (e.g. INBOX), matching +// the real Bridge connector contract. +func (conn *Dummy) MarkMessagesWithGmailLabels(_ context.Context, _ IMAPStateWrite, messageIDs []imap.MessageID, labels []string, add bool) error { + for _, label := range labels { + var mboxID imap.MailboxID + + if add { + id, mbox, created := conn.state.getOrCreateLabelMailbox(label) + mboxID = id + + if created { + conn.pushUpdate(imap.NewMailboxCreated(mbox)) + } + } else { + id, ok := conn.state.getLabelMailboxID(label) + if !ok { + // Removing a label that was never applied is a no-op. + continue + } + + mboxID = id + } + + for _, messageID := range messageIDs { + if add { + conn.state.addGmailLabel(messageID, label) + conn.state.addMessageToMailbox(messageID, mboxID) + } else { + conn.state.removeGmailLabel(messageID, label) + conn.state.removeMessageFromMailbox(messageID, mboxID) + } + + conn.pushUpdate(imap.NewMessageMailboxesUpdated( + messageID, + conn.state.getMailboxIDs(messageID), + conn.state.getMessageFlags(messageID), + )) + } + } + + return nil +} + +func (conn *Dummy) GetGmailLabels(_ context.Context, messageID imap.MessageID) ([]string, error) { + return conn.state.getGmailLabels(messageID), nil +} + +func (conn *Dummy) GetGmailLabelMailboxID(_ context.Context, label string) (imap.MailboxID, bool) { + return conn.state.getLabelMailboxID(label) +} + func (conn *Dummy) Sync(ctx context.Context) error { for _, mailbox := range conn.state.getMailboxes() { update := imap.NewMailboxCreated(mailbox) diff --git a/connector/dummy_state.go b/connector/dummy_state.go index 378d8459..2667bd2d 100644 --- a/connector/dummy_state.go +++ b/connector/dummy_state.go @@ -2,6 +2,7 @@ package connector import ( "context" + "sort" "sync" "time" @@ -18,6 +19,11 @@ type dummyState struct { mailboxes map[imap.MailboxID]*dummyMailbox lastIMAPID imap.IMAPID + // labelMailboxes maps a Gmail label name to the (non-exclusive) mailbox + // that backs it. Used by the X-GM-EXT-1 extension: applying a label adds + // the message to this mailbox without touching folder membership. + labelMailboxes map[string]imap.MailboxID + lock sync.RWMutex } @@ -36,16 +42,20 @@ type dummyMessage struct { flags imap.FlagSet mboxIDs map[imap.MailboxID]struct{} + + // labels holds the Gmail-style labels (X-GM-LABELS) applied to this message. + labels map[string]struct{} } func newDummyState(flags, permFlags, attrs imap.FlagSet) *dummyState { return &dummyState{ - flags: flags, - permFlags: permFlags, - attrs: attrs, - messages: make(map[imap.MessageID]*dummyMessage), - mailboxes: make(map[imap.MailboxID]*dummyMailbox), - lastIMAPID: imap.NewIMAPID(), + flags: flags, + permFlags: permFlags, + attrs: attrs, + messages: make(map[imap.MessageID]*dummyMessage), + mailboxes: make(map[imap.MailboxID]*dummyMailbox), + lastIMAPID: imap.NewIMAPID(), + labelMailboxes: make(map[string]imap.MailboxID), } } @@ -222,6 +232,7 @@ func (state *dummyState) createMessage( flags: otherFlags, date: date, mboxIDs: map[imap.MailboxID]struct{}{mboxID: {}}, + labels: make(map[string]struct{}), } return state.toMessage(messageID) @@ -245,6 +256,79 @@ func (state *dummyState) removeMessageFromMailbox(messageID imap.MessageID, mbox delete(state.messages[messageID].mboxIDs, mboxID) } +// getOrCreateLabelMailbox returns the (non-exclusive) mailbox backing the given +// Gmail label, creating it on first use. The returned bool reports whether the +// mailbox was newly created so the caller can emit a MailboxCreated update. +func (state *dummyState) getOrCreateLabelMailbox(label string) (imap.MailboxID, imap.Mailbox, bool) { + state.lock.Lock() + defer state.lock.Unlock() + + if mboxID, ok := state.labelMailboxes[label]; ok { + return mboxID, state.toMailbox(mboxID), false + } + + mboxID := imap.MailboxID(uuid.NewString()) + + // Labels are non-exclusive: applying one must not evict the message from + // its folder (e.g. INBOX), matching real Bridge behaviour. + state.mailboxes[mboxID] = &dummyMailbox{ + mboxName: []string{label}, + exclusive: false, + } + state.labelMailboxes[label] = mboxID + + return mboxID, state.toMailbox(mboxID), true +} + +func (state *dummyState) getLabelMailboxID(label string) (imap.MailboxID, bool) { + state.lock.Lock() + defer state.lock.Unlock() + + mboxID, ok := state.labelMailboxes[label] + + return mboxID, ok +} + +func (state *dummyState) addGmailLabel(messageID imap.MessageID, label string) { + state.lock.Lock() + defer state.lock.Unlock() + + msg := state.messages[messageID] + if msg.labels == nil { + msg.labels = make(map[string]struct{}) + } + + msg.labels[label] = struct{}{} +} + +func (state *dummyState) removeGmailLabel(messageID imap.MessageID, label string) { + state.lock.Lock() + defer state.lock.Unlock() + + if msg := state.messages[messageID]; msg.labels != nil { + delete(msg.labels, label) + } +} + +func (state *dummyState) getGmailLabels(messageID imap.MessageID) []string { + state.lock.Lock() + defer state.lock.Unlock() + + msg, ok := state.messages[messageID] + if !ok { + return nil + } + + labels := make([]string, 0, len(msg.labels)) + for label := range msg.labels { + labels = append(labels, label) + } + + sort.Strings(labels) + + return labels +} + func (state *dummyState) setSeen(messageID imap.MessageID, seen bool) { state.lock.Lock() defer state.lock.Unlock() diff --git a/imap/capabilities.go b/imap/capabilities.go index 51013903..e6aea644 100644 --- a/imap/capabilities.go +++ b/imap/capabilities.go @@ -11,11 +11,12 @@ const ( MOVE Capability = `MOVE` ID Capability = `ID` AUTHPLAIN Capability = `AUTH=PLAIN` + XGMEXT1 Capability = `X-GM-EXT-1` ) func IsCapabilityAvailableBeforeAuth(c Capability) bool { switch c { - case IMAP4rev1, StartTLS, IDLE, ID, AUTHPLAIN: + case IMAP4rev1, StartTLS, IDLE, ID, AUTHPLAIN, XGMEXT1: return true case UNSELECT, UIDPLUS, MOVE: return false diff --git a/imap/command/fetch.go b/imap/command/fetch.go index 6e3e14dd..5d0a04e3 100644 --- a/imap/command/fetch.go +++ b/imap/command/fetch.go @@ -165,6 +165,8 @@ func handleFetchAttribute(name rfcparser.String, p *rfcparser.Parser) (FetchAttr return handleRFC822FetchAttribute(p) case "body": return handleBodyFetchAttribute(p) + case "x": + return handleXExtensionFetchAttribute(p, name) default: return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown fetch attribute '%v'", name.Value), name.Offset) } @@ -433,6 +435,29 @@ func collectBodySectionText(p *rfcparser.Parser) (string, error) { return strings.ToLower(string(text.Value)), nil } +func handleXExtensionFetchAttribute(p *rfcparser.Parser, name rfcparser.String) (FetchAttribute, error) { + // Handle X-GM-LABELS for the Gmail X-GM-EXT-1 extension. + // parseFetchAttributeName() only collected "x" (stopped at hyphen). + // Consume the rest: "-GM-LABELS". + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown fetch attribute '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('G', 'M'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown fetch attribute '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown fetch attribute '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('L', 'A', 'B', 'E', 'L', 'S'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown fetch attribute '%v'", name.Value), name.Offset) + } + + return &FetchAttributeGmailLabels{}, nil +} + func parseHeaderList(p *rfcparser.Parser) ([]string, error) { var result []string diff --git a/imap/command/fetch_attributes.go b/imap/command/fetch_attributes.go index 03e5d10e..4f0c9260 100644 --- a/imap/command/fetch_attributes.go +++ b/imap/command/fetch_attributes.go @@ -86,6 +86,12 @@ func (f FetchAttributeUID) String() string { return "UID" } +type FetchAttributeGmailLabels struct{} + +func (f FetchAttributeGmailLabels) String() string { + return "X-GM-LABELS" +} + type BodySection interface { String() string } diff --git a/imap/command/gmail_labels_test.go b/imap/command/gmail_labels_test.go new file mode 100644 index 00000000..1fbca502 --- /dev/null +++ b/imap/command/gmail_labels_test.go @@ -0,0 +1,202 @@ +package command + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// These tests cover parsing of the X-GM-EXT-1 Gmail label extension across the +// STORE, FETCH and SEARCH commands. The extension is what Paperless-NGX (and +// other Gmail-style clients) use to tag mail over IMAP. + +// --- STORE +/-/= X-GM-LABELS ---------------------------------------------- + +func TestParser_StoreCommandAddGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionAddFlags, + Flags: []string{"Label1", "Label2"}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 +X-GM-LABELS ("Label1" "Label2")`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_StoreCommandRemoveGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionRemFlags, + Flags: []string{"Label1"}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 -X-GM-LABELS ("Label1")`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +// STORE X-GM-LABELS without a +/- prefix parses as StoreActionSetFlags. Note +// that the session handler currently treats anything other than AddFlags as a +// removal (see handleStoreGmailLabels) — this test pins the parse only. +func TestParser_StoreCommandSetGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionSetFlags, + Flags: []string{"Label1"}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 X-GM-LABELS ("Label1")`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +// Python's imaplib sends the label without wrapping it in parentheses. This is +// the exact form Paperless-NGX emits, so it must parse (regression for the +// "accept bare labels" fix). +func TestParser_StoreCommandAddGmailLabelsBare(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionAddFlags, + Flags: []string{"Paperless"}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 +X-GM-LABELS Paperless`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_StoreCommandGmailLabelsWithSpaces(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionAddFlags, + Flags: []string{"Label With Spaces", "Other"}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 +X-GM-LABELS ("Label With Spaces" "Other")`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_StoreCommandGmailLabelsSystemLabel(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionAddFlags, + Flags: []string{`\Inbox`}, + Silent: false, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 +X-GM-LABELS (\Inbox)`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_StoreCommandGmailLabelsSilent(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Store{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Action: StoreActionAddFlags, + Flags: []string{"Label1"}, + Silent: true, + DataItem: StoreDataItemGmailLabels, + }} + + cmd, err := testParseCommand(`tag STORE 1 +X-GM-LABELS.SILENT ("Label1")`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +// --- FETCH X-GM-LABELS ------------------------------------------------------ + +func TestParser_FetchCommandGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Fetch{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Attributes: []FetchAttribute{ + &FetchAttributeGmailLabels{}, + }, + }} + + cmd, err := testParseCommand(`tag FETCH 1 X-GM-LABELS`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_FetchCommandGmailLabelsWithFlags(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Fetch{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Attributes: []FetchAttribute{ + &FetchAttributeFlags{}, + &FetchAttributeGmailLabels{}, + }, + }} + + cmd, err := testParseCommand(`tag FETCH 1 (FLAGS X-GM-LABELS)`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_FetchCommandGmailLabelsLowercase(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Fetch{ + SeqSet: []SeqRange{{Begin: 1, End: 1}}, + Attributes: []FetchAttribute{ + &FetchAttributeGmailLabels{}, + }, + }} + + cmd, err := testParseCommand(`tag FETCH 1 (x-gm-labels)`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +// --- SEARCH X-GM-LABELS ----------------------------------------------------- + +func TestParser_SearchCommandGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Search{ + Charset: "", + Keys: []SearchKey{ + &SearchKeyGmailLabels{Value: "Paperless"}, + }, + }} + + cmd, err := testParseCommand(`tag SEARCH X-GM-LABELS "Paperless"`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +func TestParser_SearchCommandGmailLabelsAtom(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Search{ + Charset: "", + Keys: []SearchKey{ + &SearchKeyGmailLabels{Value: "Paperless"}, + }, + }} + + cmd, err := testParseCommand(`tag SEARCH X-GM-LABELS Paperless`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} + +// NOT X-GM-LABELS "" is exactly how Paperless-NGX excludes already-tagged +// mail from its search criteria, so the negated form must parse. +func TestParser_SearchCommandNotGmailLabels(t *testing.T) { + expected := Command{Tag: "tag", Payload: &Search{ + Charset: "", + Keys: []SearchKey{ + &SearchKeyNot{Key: &SearchKeyGmailLabels{Value: "Paperless"}}, + }, + }} + + cmd, err := testParseCommand(`tag SEARCH NOT X-GM-LABELS "Paperless"`) + require.NoError(t, err) + require.Equal(t, expected, cmd) +} diff --git a/imap/command/search.go b/imap/command/search.go index 5f754566..352a1f89 100644 --- a/imap/command/search.go +++ b/imap/command/search.go @@ -470,6 +470,9 @@ func handleSearchKey(keyword rfcparser.String, p *rfcparser.Parser) (SearchKey, case "undraft": return &SearchKeyUndraft{}, nil + case "x": + return handleXExtensionSearchKey(p, keyword) + default: return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown search key '%v'", keyword.Value), keyword.Offset) } @@ -511,3 +514,28 @@ func parseStringKeyAtom(p *rfcparser.Parser) (string, error) { return p.ParseAtom() } + +func handleXExtensionSearchKey(p *rfcparser.Parser, name rfcparser.String) (SearchKey, error) { + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown search key '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('G', 'M'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown search key '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown search key '%v'", name.Value), name.Offset) + } + + if err := p.ConsumeBytesFold('L', 'A', 'B', 'E', 'L', 'S'); err != nil { + return nil, p.MakeErrorAtOffset(fmt.Sprintf("unknown search key '%v'", name.Value), name.Offset) + } + + value, err := parseStringKeyAString(p) + if err != nil { + return nil, err + } + + return &SearchKeyGmailLabels{Value: value}, nil +} diff --git a/imap/command/search_keys.go b/imap/command/search_keys.go index c54a440b..4b5fa4cf 100644 --- a/imap/command/search_keys.go +++ b/imap/command/search_keys.go @@ -418,6 +418,18 @@ func (s SearchKeyList) SanitizedString() string { }), "")) } +type SearchKeyGmailLabels struct { + Value string +} + +func (s SearchKeyGmailLabels) String() string { + return fmt.Sprintf("X-GM-LABELS %v", s.Value) +} + +func (s SearchKeyGmailLabels) SanitizedString() string { + return fmt.Sprintf("X-GM-LABELS %v", sanitizeString(s.Value)) +} + type SearchKeySeqSet struct { SeqSet []SeqRange } diff --git a/imap/command/store.go b/imap/command/store.go index a9dcc867..0df82b34 100644 --- a/imap/command/store.go +++ b/imap/command/store.go @@ -27,11 +27,20 @@ func (s StoreAction) String() string { } } +// StoreDataItem distinguishes between standard IMAP FLAGS and Gmail X-GM-LABELS. +type StoreDataItem int + +const ( + StoreDataItemFlags StoreDataItem = iota // Standard IMAP FLAGS + StoreDataItemGmailLabels // Gmail X-GM-LABELS extension +) + type Store struct { - SeqSet []SeqRange - Action StoreAction - Flags []string - Silent bool + SeqSet []SeqRange + Action StoreAction + Flags []string + Silent bool + DataItem StoreDataItem } func (s Store) String() string { @@ -40,7 +49,19 @@ func (s Store) String() string { silentStr = ".SILENT" } - return fmt.Sprintf("STORE %v %v%v %v", s.SeqSet, s.Action.String(), silentStr, s.Flags) + dataItemStr := s.Action.String() + if s.DataItem == StoreDataItemGmailLabels { + switch s.Action { + case StoreActionAddFlags: + dataItemStr = "+X-GM-LABELS" + case StoreActionRemFlags: + dataItemStr = "-X-GM-LABELS" + default: + dataItemStr = "X-GM-LABELS" + } + } + + return fmt.Sprintf("STORE %v %v%v %v", s.SeqSet, dataItemStr, silentStr, s.Flags) } func (s Store) SanitizedString() string { @@ -54,6 +75,9 @@ func (StoreCommandParser) FromParser(p *rfcparser.Parser) (Payload, error) { // store = "STORE" SP sequence-set SP store-att-flags // store-att-flags = (["+" / "-"] "FLAGS" [".SILENT"]) SP // (flag-list / (flag *(SP flag))) + // Gmail extension: + // store-att-flags =/ (["+" / "-"] "X-GM-LABELS" [".SILENT"]) SP + // (label-list) if err := p.Consume(rfcparser.TokenTypeSP, "expected space after command"); err != nil { return nil, err } @@ -83,8 +107,46 @@ func (StoreCommandParser) FromParser(p *rfcparser.Parser) (Payload, error) { action = StoreActionAddFlags } - if err := p.ConsumeBytesFold('F', 'L', 'A', 'G', 'S'); err != nil { - return nil, err + // Determine data item: FLAGS or X-GM-LABELS. + // Peek at current byte to decide which path to take. + currentByte := rfcparser.ByteToLower(p.CurrentToken().Value) + + var dataItem StoreDataItem + + switch currentByte { + case 'f': + // Standard FLAGS data item. + if err := p.ConsumeBytesFold('F', 'L', 'A', 'G', 'S'); err != nil { + return nil, err + } + + dataItem = StoreDataItemFlags + case 'x': + // Gmail X-GM-LABELS data item. + // Consume "X-GM-LABELS" character-by-character. + if err := p.ConsumeBytesFold('X'); err != nil { + return nil, err + } + + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, err + } + + if err := p.ConsumeBytesFold('G', 'M'); err != nil { + return nil, err + } + + if err := p.ConsumeBytesFold('-'); err != nil { + return nil, err + } + + if err := p.ConsumeBytesFold('L', 'A', 'B', 'E', 'L', 'S'); err != nil { + return nil, err + } + + dataItem = StoreDataItemGmailLabels + default: + return nil, p.MakeError("expected FLAGS or X-GM-LABELS") } var silent bool @@ -99,20 +161,28 @@ func (StoreCommandParser) FromParser(p *rfcparser.Parser) (Payload, error) { silent = true } - if err := p.Consume(rfcparser.TokenTypeSP, "expected space after FLAGS"); err != nil { + if err := p.Consume(rfcparser.TokenTypeSP, "expected space after data item"); err != nil { return nil, err } - flags, err := parseStoreFlags(p) + var values []string + + if dataItem == StoreDataItemGmailLabels { + values, err = parseGmailLabelList(p) + } else { + values, err = parseStoreFlags(p) + } + if err != nil { return nil, err } return &Store{ - SeqSet: seqSet, - Action: action, - Flags: flags, - Silent: silent, + SeqSet: seqSet, + Action: action, + Flags: values, + Silent: silent, + DataItem: dataItem, }, nil } @@ -155,3 +225,80 @@ func parseStoreFlags(p *rfcparser.Parser) ([]string, error) { return flags, nil } + +// parseGmailLabelList parses a Gmail label list. It accepts either a parenthesized list +// ("Label1" "Label With Spaces" Label3) or a single bare label without parentheses. +func parseGmailLabelList(p *rfcparser.Parser) ([]string, error) { + // If it starts with '(', parse as a parenthesized list. + if ok, err := p.Matches(rfcparser.TokenTypeLParen); err != nil { + return nil, err + } else if ok { + var labels []string + + if !p.Check(rfcparser.TokenTypeRParen) { + label, err := parseGmailLabel(p) + if err != nil { + return nil, err + } + + labels = append(labels, label) + + for { + if ok, err := p.Matches(rfcparser.TokenTypeSP); err != nil { + return nil, err + } else if !ok { + break + } + + label, err := parseGmailLabel(p) + if err != nil { + return nil, err + } + + labels = append(labels, label) + } + } + + if err := p.Consume(rfcparser.TokenTypeRParen, "expected ')' at end of Gmail label list"); err != nil { + return nil, err + } + + return labels, nil + } + + // No parenthesis — parse a single bare label. + label, err := parseGmailLabel(p) + if err != nil { + return nil, err + } + + return []string{label}, nil +} + +// parseGmailLabel parses a single Gmail label, which can be a quoted string or an atom. +func parseGmailLabel(p *rfcparser.Parser) (string, error) { + // Try quoted string first (handles labels with spaces). + if p.Check(rfcparser.TokenTypeDQuote) { + quoted, err := p.ParseQuoted() + if err != nil { + return "", err + } + + return quoted.Value, nil + } + + // Try backslash-prefixed system label (e.g., \Inbox). + if hasBackslash, err := p.Matches(rfcparser.TokenTypeBackslash); err != nil { + return "", err + } else if hasBackslash { + atom, err := p.ParseAtom() + if err != nil { + return "", err + } + + return fmt.Sprintf("\\%v", atom), nil + } + + // Fall back to atom (plain label name without spaces). + return p.ParseAtom() +} diff --git a/internal/backend/state_connector_impl.go b/internal/backend/state_connector_impl.go index 9320ccf0..76b15ef0 100644 --- a/internal/backend/state_connector_impl.go +++ b/internal/backend/state_connector_impl.go @@ -208,6 +208,36 @@ func (sc *stateConnectorImpl) SetMessagesForwarded( return cache.stateUpdates, nil } +func (sc *stateConnectorImpl) SetGmailLabels( + ctx context.Context, + tx db.Transaction, + messageIDs []imap.MessageID, + labels []string, + add bool, +) ([]state.Update, error) { + ctx = sc.newContextWithMetadata(ctx) + + cache := sc.newDBIMAPWrite(tx) + + if err := sc.connector.MarkMessagesWithGmailLabels(ctx, &cache, messageIDs, labels, add); err != nil { + return nil, err + } + + return cache.stateUpdates, nil +} + +func (sc *stateConnectorImpl) GetGmailLabels(ctx context.Context, messageID imap.MessageID) ([]string, error) { + ctx = sc.newContextWithMetadata(ctx) + + return sc.connector.GetGmailLabels(ctx, messageID) +} + +func (sc *stateConnectorImpl) GetGmailLabelMailboxID(ctx context.Context, label string) (imap.MailboxID, bool) { + ctx = sc.newContextWithMetadata(ctx) + + return sc.connector.GetGmailLabelMailboxID(ctx, label) +} + func (sc *stateConnectorImpl) getMetadataValue(key string) any { v, ok := sc.metadata[key] if !ok { diff --git a/internal/response/item_gmail_labels.go b/internal/response/item_gmail_labels.go new file mode 100644 index 00000000..f57522c0 --- /dev/null +++ b/internal/response/item_gmail_labels.go @@ -0,0 +1,27 @@ +package response + +import ( + "fmt" + "strings" +) + +type itemGmailLabels struct { + labels []string +} + +func ItemGmailLabels(labels []string) *itemGmailLabels { + return &itemGmailLabels{labels: labels} +} + +func (g *itemGmailLabels) String() string { + if len(g.labels) == 0 { + return "X-GM-LABELS ()" + } + + quoted := make([]string, len(g.labels)) + for i, label := range g.labels { + quoted[i] = fmt.Sprintf("%q", label) + } + + return fmt.Sprintf("X-GM-LABELS (%v)", strings.Join(quoted, " ")) +} diff --git a/internal/response/item_gmail_labels_test.go b/internal/response/item_gmail_labels_test.go new file mode 100644 index 00000000..e0c7c4a3 --- /dev/null +++ b/internal/response/item_gmail_labels_test.go @@ -0,0 +1,50 @@ +package response + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestItemGmailLabelsEmpty(t *testing.T) { + assert.Equal(t, "X-GM-LABELS ()", ItemGmailLabels(nil).String()) + assert.Equal(t, "X-GM-LABELS ()", ItemGmailLabels([]string{}).String()) +} + +func TestItemGmailLabelsSingle(t *testing.T) { + assert.Equal(t, `X-GM-LABELS ("Paperless")`, ItemGmailLabels([]string{"Paperless"}).String()) +} + +func TestItemGmailLabelsMultiple(t *testing.T) { + assert.Equal( + t, + `X-GM-LABELS ("Label1" "Label2" "Label3")`, + ItemGmailLabels([]string{"Label1", "Label2", "Label3"}).String(), + ) +} + +func TestItemGmailLabelsWithSpaces(t *testing.T) { + assert.Equal( + t, + `X-GM-LABELS ("Label With Spaces" "Another One")`, + ItemGmailLabels([]string{"Label With Spaces", "Another One"}).String(), + ) +} + +// A label containing a double quote must be escaped so the response stays a +// valid IMAP quoted string. +func TestItemGmailLabelsQuoting(t *testing.T) { + assert.Equal( + t, + `X-GM-LABELS ("a\"b")`, + ItemGmailLabels([]string{`a"b`}).String(), + ) +} + +func TestItemGmailLabelsSystemLabel(t *testing.T) { + assert.Equal( + t, + `X-GM-LABELS ("\\Inbox")`, + ItemGmailLabels([]string{`\Inbox`}).String(), + ) +} diff --git a/internal/session/gmail_ext_gate.go b/internal/session/gmail_ext_gate.go new file mode 100644 index 00000000..5a9e6c2e --- /dev/null +++ b/internal/session/gmail_ext_gate.go @@ -0,0 +1,50 @@ +package session + +import ( + "errors" + + "github.com/ProtonMail/gluon/imap/command" +) + +// errGmailExtensionDisabled is returned when a client issues an X-GM-LABELS +// command while the Gmail X-GM-EXT-1 extension is not enabled. In that case the +// capability is not advertised, so a compliant client will not send these +// commands; this guards against non-compliant clients that do anyway. +var errGmailExtensionDisabled = errors.New("X-GM-EXT-1 extension is not enabled") + +// fetchHasGmailLabels reports whether a FETCH command requests the X-GM-LABELS attribute. +func fetchHasGmailLabels(cmd *command.Fetch) bool { + for _, attr := range cmd.Attributes { + if _, ok := attr.(*command.FetchAttributeGmailLabels); ok { + return true + } + } + + return false +} + +// searchKeysHaveGmailLabels reports whether any (possibly nested) SEARCH key uses X-GM-LABELS. +func searchKeysHaveGmailLabels(keys []command.SearchKey) bool { + for _, key := range keys { + if searchKeyHasGmailLabels(key) { + return true + } + } + + return false +} + +func searchKeyHasGmailLabels(key command.SearchKey) bool { + switch k := key.(type) { + case *command.SearchKeyGmailLabels: + return true + case *command.SearchKeyNot: + return searchKeyHasGmailLabels(k.Key) + case *command.SearchKeyOr: + return searchKeyHasGmailLabels(k.Key1) || searchKeyHasGmailLabels(k.Key2) + case *command.SearchKeyList: + return searchKeysHaveGmailLabels(k.Keys) + default: + return false + } +} diff --git a/internal/session/handle_fetch.go b/internal/session/handle_fetch.go index 4977fb60..b5deba62 100644 --- a/internal/session/handle_fetch.go +++ b/internal/session/handle_fetch.go @@ -21,6 +21,10 @@ func (s *Session) handleFetch(ctx context.Context, tag string, cmd *command.Fetc defer profiling.Stop(ctx, profiling.CmdTypeFetch) } + if !s.enableGmailExtension && fetchHasGmailLabels(cmd) { + return response.Bad(tag).WithError(errGmailExtensionDisabled), nil + } + if err := mailbox.Fetch(ctx, cmd, ch); errors.Is(err, state.ErrNoSuchMessage) { return response.Bad(tag).WithError(err), nil } else if err != nil { diff --git a/internal/session/handle_search.go b/internal/session/handle_search.go index bd3a467b..cdcea8a6 100644 --- a/internal/session/handle_search.go +++ b/internal/session/handle_search.go @@ -21,6 +21,10 @@ func (s *Session) handleSearch(ctx context.Context, tag string, cmd *command.Sea defer profiling.Stop(ctx, profiling.CmdTypeSearch) } + if !s.enableGmailExtension && searchKeysHaveGmailLabels(cmd.Keys) { + return response.Bad(tag).WithError(errGmailExtensionDisabled), nil + } + var decoder *encoding.Decoder if len(cmd.Charset) != 0 { diff --git a/internal/session/handle_store.go b/internal/session/handle_store.go index a8c7b1d7..ff08ce95 100644 --- a/internal/session/handle_store.go +++ b/internal/session/handle_store.go @@ -30,6 +30,15 @@ func (s *Session) handleStore(ctx context.Context, tag string, cmd *command.Stor return nil, ErrReadOnly } + // Route based on data item type: standard FLAGS vs Gmail X-GM-LABELS. + if cmd.DataItem == command.StoreDataItemGmailLabels { + if !s.enableGmailExtension { + return response.Bad(tag).WithError(errGmailExtensionDisabled), nil + } + + return s.handleStoreGmailLabels(ctx, tag, cmd, mailbox, ch) + } + flags, err := validateStoreFlags(cmd.Flags) if err != nil { return response.Bad(tag).WithError(err), nil @@ -58,3 +67,21 @@ func (s *Session) handleStore(ctx context.Context, tag string, cmd *command.Stor WithItems(items...). WithMessage(okMessage(ctx)), nil } + +// handleStoreGmailLabels handles STORE commands with the X-GM-LABELS data item. +// This translates Gmail label operations into connector label operations. +func (s *Session) handleStoreGmailLabels(ctx context.Context, tag string, cmd *command.Store, mailbox *state.Mailbox, ch chan response.Response) (response.Response, error) { + add := cmd.Action == command.StoreActionAddFlags + + if err := mailbox.StoreGmailLabels(ctx, cmd.SeqSet, cmd.Flags, add); errors.Is(err, state.ErrNoSuchMessage) { + return response.Bad(tag).WithError(err), nil + } else if err != nil { + return nil, err + } + + if err := flush(ctx, mailbox, false, ch); err != nil { + return nil, err + } + + return response.Ok(tag).WithMessage(okMessage(ctx)), nil +} diff --git a/internal/session/session.go b/internal/session/session.go index 0c02c647..b238bd18 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -92,6 +92,9 @@ type Session struct { // disableIMAPAuthenticate disables the IMAP AUTHENTICATE command (client can then only authenticate using LOGIN). disableIMAPAuthenticate bool + // enableGmailExtension enables the non-standard Gmail X-GM-EXT-1 extension (capability advertisement + X-GM-LABELS handling). + enableGmailExtension bool + // panicHandler The panic handler. panicHandler async.PanicHandler @@ -112,6 +115,7 @@ func New( eventCh chan<- events.Event, idleBulkTime time.Duration, disableIMAPAuthenticate bool, + enableGmailExtension bool, panicHandler async.PanicHandler, featureFlagProvider unleash.FeatureFlagValueProvider, ) *Session { @@ -128,6 +132,10 @@ func New( caps = append(caps, imap.AUTHPLAIN) } + if enableGmailExtension { + caps = append(caps, imap.XGMEXT1) + } + return &Session{ conn: conn, inputCollector: inputCollector, @@ -141,6 +149,7 @@ func New( cmdProfilerBuilder: profiler, handleWG: async.MakeWaitGroup(panicHandler), disableIMAPAuthenticate: disableIMAPAuthenticate, + enableGmailExtension: enableGmailExtension, panicHandler: panicHandler, log: logrus.WithField("pkg", "gluon/session").WithField("session", sessionID), featureFlagProvider: featureFlagProvider, diff --git a/internal/state/actions.go b/internal/state/actions.go index ee16a63c..00b49cb0 100644 --- a/internal/state/actions.go +++ b/internal/state/actions.go @@ -643,3 +643,47 @@ func (state *State) actionSetMessageFlags(ctx context.Context, return state.applyMessageFlagsSet(ctx, tx, internalMessageIDs, setFlags) } + +// actionAddGmailLabels adds Gmail-style labels to messages via the connector. +func (state *State) actionAddGmailLabels( + ctx context.Context, + tx db.Transaction, + messages []snapMsgWithSeq, + labels []string, +) ([]Update, error) { + messageIDs := make([]imap.MessageID, 0, len(messages)) + + for _, sm := range messages { + if !ids.IsRecoveredRemoteMessageID(sm.ID.RemoteID) { + messageIDs = append(messageIDs, sm.ID.RemoteID) + } + } + + if len(messageIDs) == 0 { + return nil, nil + } + + return state.user.GetRemote().SetGmailLabels(ctx, tx, messageIDs, labels, true) +} + +// actionRemoveGmailLabels removes Gmail-style labels from messages via the connector. +func (state *State) actionRemoveGmailLabels( + ctx context.Context, + tx db.Transaction, + messages []snapMsgWithSeq, + labels []string, +) ([]Update, error) { + messageIDs := make([]imap.MessageID, 0, len(messages)) + + for _, sm := range messages { + if !ids.IsRecoveredRemoteMessageID(sm.ID.RemoteID) { + messageIDs = append(messageIDs, sm.ID.RemoteID) + } + } + + if len(messageIDs) == 0 { + return nil, nil + } + + return state.user.GetRemote().SetGmailLabels(ctx, tx, messageIDs, labels, false) +} diff --git a/internal/state/connector.go b/internal/state/connector.go index b7d01c73..8061d304 100644 --- a/internal/state/connector.go +++ b/internal/state/connector.go @@ -83,4 +83,16 @@ type Connector interface { // SetMessagesForwarded marks the message with the given ID as forwarded. SetMessagesForwarded(ctx context.Context, tx db.Transaction, messageIDs []imap.MessageID, forwarded bool) ([]Update, error) + + // SetGmailLabels adds or removes Gmail-style labels on the given messages. + // This is part of the X-GM-EXT-1 extension for compatibility with Gmail IMAP clients. + SetGmailLabels(ctx context.Context, tx db.Transaction, messageIDs []imap.MessageID, labels []string, add bool) ([]Update, error) + + // GetGmailLabels retrieves the Gmail-style label names for the given message. + // Note: this can get called from different go routines. + GetGmailLabels(ctx context.Context, messageID imap.MessageID) ([]string, error) + + // GetGmailLabelMailboxID returns the IMAP mailbox ID for a given Gmail label name. + // Used for efficient SEARCH X-GM-LABELS operations via local DB lookups. + GetGmailLabelMailboxID(ctx context.Context, label string) (imap.MailboxID, bool) } diff --git a/internal/state/mailbox.go b/internal/state/mailbox.go index 1121f418..fbbb3031 100644 --- a/internal/state/mailbox.go +++ b/internal/state/mailbox.go @@ -401,6 +401,23 @@ func (m *Mailbox) Store(ctx context.Context, seqSet []command.SeqRange, action c }) } +// StoreGmailLabels handles STORE X-GM-LABELS commands by routing to the connector's +// Gmail label handler. Labels are applied/removed without affecting folder membership. +func (m *Mailbox) StoreGmailLabels(ctx context.Context, seqSet []command.SeqRange, labels []string, add bool) error { + messages, err := m.snap.getMessagesInRange(ctx, seqSet) + if err != nil { + return err + } + + return stateDBWrite(ctx, m.state, func(ctx context.Context, tx db.Transaction) ([]Update, error) { + if add { + return m.state.actionAddGmailLabels(ctx, tx, messages, labels) + } + + return m.state.actionRemoveGmailLabels(ctx, tx, messages, labels) + }) +} + func (m *Mailbox) Expunge(ctx context.Context, seq []command.SeqRange) error { var msgIDs []db.MessageIDPair diff --git a/internal/state/mailbox_fetch.go b/internal/state/mailbox_fetch.go index 8a9a7fa5..93872125 100644 --- a/internal/state/mailbox_fetch.go +++ b/internal/state/mailbox_fetch.go @@ -92,6 +92,17 @@ func (m *Mailbox) Fetch(ctx context.Context, cmd *command.Fetch, ch chan respons return fetchAttributeBodySection(attribute, literal) } + operations = append(operations, op) + case *command.FetchAttributeGmailLabels: + op := func(msg snapMsgWithSeq, _ *db.Message, _ []byte) (response.Item, error) { + labels, err := m.state.user.GetRemote().GetGmailLabels(ctx, msg.ID.RemoteID) + if err != nil { + return response.ItemGmailLabels(nil), nil + } + + return response.ItemGmailLabels(labels), nil + } + operations = append(operations, op) } } diff --git a/internal/state/mailbox_search.go b/internal/state/mailbox_search.go index 2fa5dd41..99bc5d38 100644 --- a/internal/state/mailbox_search.go +++ b/internal/state/mailbox_search.go @@ -36,7 +36,7 @@ func (m *Mailbox) Search(ctx context.Context, keys []command.SearchKey, decoder } } - op, err := buildSearchOpListWithKeys(m, keys, decoder) + op, err := buildSearchOpListWithKeys(ctx, m, keys, decoder) if err != nil { return nil, err } @@ -84,7 +84,7 @@ func (m *Mailbox) Search(ctx context.Context, keys []command.SearchKey, decoder } func buildSearchData(ctx context.Context, m *Mailbox, op *buildSearchOpResult, message snapMsgWithSeq) (searchData, error) { - data := searchData{message: message} + data := searchData{ctx: ctx, message: message} if op.needsMessage { if err := stateDBRead(ctx, m.state, func(ctx context.Context, client db.ReadOnly) error { @@ -137,6 +137,7 @@ func applySearch(ctx context.Context, m *Mailbox, msg snapMsgWithSeq, searchOp * } type searchData struct { + ctx context.Context message snapMsgWithSeq literal []byte dbMessage struct { @@ -206,7 +207,7 @@ func newBuildSearchOpResult(op searchOp, needs ...searchOpResultOption) *buildSe return r } -func buildSearchOp(m *Mailbox, key command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { +func buildSearchOp(ctx context.Context, m *Mailbox, key command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { switch key := key.(type) { case *command.SearchKeyAll: return buildSearchOpAll() @@ -251,7 +252,7 @@ func buildSearchOp(m *Mailbox, key command.SearchKey, decoder *encoding.Decoder) return buildSearchOpNew() case *command.SearchKeyNot: - return buildSearchOpNot(m, key, decoder) + return buildSearchOpNot(ctx, m, key, decoder) case *command.SearchKeyOld: return buildSearchOpOld() @@ -260,7 +261,7 @@ func buildSearchOp(m *Mailbox, key command.SearchKey, decoder *encoding.Decoder) return buildSearchOpOn(key) case *command.SearchKeyOr: - return buildSearchOpOr(m, key, decoder) + return buildSearchOpOr(ctx, m, key, decoder) case *command.SearchKeyRecent: return buildSearchOpRecent() @@ -316,8 +317,11 @@ func buildSearchOp(m *Mailbox, key command.SearchKey, decoder *encoding.Decoder) case *command.SearchKeySeqSet: return buildSearchOpSeqSet(m, key) + case *command.SearchKeyGmailLabels: + return buildSearchOpGmailLabels(ctx, m, key) + case *command.SearchKeyList: - return buildSearchOpList(m, key.Keys, decoder) + return buildSearchOpList(ctx, m, key.Keys, decoder) default: return nil, fmt.Errorf("bad search keyword") @@ -485,8 +489,8 @@ func buildSearchOpNew() (*buildSearchOpResult, error) { return newBuildSearchOpResult(op), nil } -func buildSearchOpNot(m *Mailbox, key *command.SearchKeyNot, decoder *encoding.Decoder) (*buildSearchOpResult, error) { - toNegateOpResult, err := buildSearchOp(m, key.Key, decoder) +func buildSearchOpNot(ctx context.Context, m *Mailbox, key *command.SearchKeyNot, decoder *encoding.Decoder) (*buildSearchOpResult, error) { + toNegateOpResult, err := buildSearchOp(ctx, m, key.Key, decoder) if err != nil { return nil, err } @@ -524,13 +528,13 @@ func buildSearchOpOn(key *command.SearchKeyOn) (*buildSearchOpResult, error) { return newBuildSearchOpResult(op, needsDBMessage()), nil } -func buildSearchOpOr(m *Mailbox, key *command.SearchKeyOr, decoder *encoding.Decoder) (*buildSearchOpResult, error) { - leftOp, err := buildSearchOp(m, key.Key1, decoder) +func buildSearchOpOr(ctx context.Context, m *Mailbox, key *command.SearchKeyOr, decoder *encoding.Decoder) (*buildSearchOpResult, error) { + leftOp, err := buildSearchOp(ctx, m, key.Key1, decoder) if err != nil { return nil, err } - rightOp, err := buildSearchOp(m, key.Key2, decoder) + rightOp, err := buildSearchOp(ctx, m, key.Key2, decoder) if err != nil { return nil, err } @@ -789,17 +793,17 @@ func buildSearchOpSeqSet(m *Mailbox, key *command.SearchKeySeqSet) (*buildSearch return newBuildSearchOpResult(op), nil } -func buildSearchOpList(m *Mailbox, keys []command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { - return buildSearchOpListWithKeys(m, keys, decoder) +func buildSearchOpList(ctx context.Context, m *Mailbox, keys []command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { + return buildSearchOpListWithKeys(ctx, m, keys, decoder) } -func buildSearchOpListWithKeys(m *Mailbox, opKeys []command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { +func buildSearchOpListWithKeys(ctx context.Context, m *Mailbox, opKeys []command.SearchKey, decoder *encoding.Decoder) (*buildSearchOpResult, error) { ops := make([]searchOp, 0, len(opKeys)) opResult := newBuildSearchOpResult(nil) for _, opKey := range opKeys { - result, err := buildSearchOp(m, opKey, decoder) + result, err := buildSearchOp(ctx, m, opKey, decoder) if err != nil { return nil, err } @@ -830,6 +834,48 @@ func buildSearchOpListWithKeys(m *Mailbox, opKeys []command.SearchKey, decoder * return opResult, nil } +func buildSearchOpGmailLabels(ctx context.Context, m *Mailbox, key *command.SearchKeyGmailLabels) (*buildSearchOpResult, error) { + noMatch := func(s *searchData) (bool, error) { return false, nil } + + // Get the mailbox ID for this label from the connector (reads in-memory label cache). + remoteMailboxID, ok := m.state.user.GetRemote().GetGmailLabelMailboxID(ctx, key.Value) + if !ok { + return newBuildSearchOpResult(noMatch), nil + } + + // Query local DB for all messages in the label mailbox (one query). + var memberSet map[imap.InternalMessageID]struct{} + + if err := stateDBRead(ctx, m.state, func(ctx context.Context, client db.ReadOnly) error { + internalMboxID, err := client.GetMailboxIDFromRemoteID(ctx, remoteMailboxID) + if err != nil { + return err + } + + pairs, err := client.GetMailboxMessageIDPairs(ctx, internalMboxID) + if err != nil { + return err + } + + memberSet = make(map[imap.InternalMessageID]struct{}, len(pairs)) + for _, p := range pairs { + memberSet[p.InternalID] = struct{}{} + } + + return nil + }); err != nil { + return newBuildSearchOpResult(noMatch), nil + } + + // O(1) set lookup per message — no API calls. + op := func(s *searchData) (bool, error) { + _, exists := memberSet[s.message.ID.InternalID] + return exists, nil + } + + return newBuildSearchOpResult(op), nil +} + func convertToDateWithoutTZ(t time.Time) time.Time { return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) } diff --git a/option.go b/option.go index 9459df8f..2b09ac8d 100644 --- a/option.go +++ b/option.go @@ -234,6 +234,19 @@ func WithDisableIMAPAuthenticate() Option { return &withDisableIMAPAuthenticate{} } +type withGmailExtension struct{} + +func (withGmailExtension) config(builder *serverBuilder) { + builder.enableGmailExtension = true +} + +// WithGmailExtension enables the non-standard Gmail X-GM-EXT-1 IMAP extension +// (X-GM-LABELS via STORE/FETCH/SEARCH). When not set, the capability is not +// advertised and the X-GM-LABELS commands are rejected. +func WithGmailExtension() Option { + return &withGmailExtension{} +} + type withUIDValidityGenerator struct { generator imap.UIDValidityGenerator } diff --git a/server.go b/server.go index a742f163..0a918f19 100644 --- a/server.go +++ b/server.go @@ -93,6 +93,9 @@ type Server struct { // disableIMAPAuthenticate disables the IMAP AUTHENTICATE command (client can then only authenticate using LOGIN). disableIMAPAuthenticate bool + // enableGmailExtension enables the non-standard Gmail X-GM-EXT-1 extension (capability advertisement + X-GM-LABELS handling). + enableGmailExtension bool + uidValidityGenerator imap.UIDValidityGenerator panicHandler async.PanicHandler @@ -377,7 +380,7 @@ func (s *Server) addSession(ctx context.Context, conn net.Conn) (*session.Sessio nextID := s.getNextID() - s.sessions[nextID] = session.New(conn, s.backend, nextID, s.versionInfo, s.cmdExecProfBuilder, s.newEventCh(ctx), s.idleBulkTime, s.disableIMAPAuthenticate, s.panicHandler, s.featureFlagProvider) + s.sessions[nextID] = session.New(conn, s.backend, nextID, s.versionInfo, s.cmdExecProfBuilder, s.newEventCh(ctx), s.idleBulkTime, s.disableIMAPAuthenticate, s.enableGmailExtension, s.panicHandler, s.featureFlagProvider) if s.tlsConfig != nil { s.sessions[nextID].SetTLSConfig(s.tlsConfig) diff --git a/tests/capability_test.go b/tests/capability_test.go index 4afb52a3..7014b5dc 100644 --- a/tests/capability_test.go +++ b/tests/capability_test.go @@ -33,3 +33,19 @@ func TestCapabilityAuthenticateDisabled(t *testing.T) { c.S("A003 OK CAPABILITY") }) } + +// With the Gmail extension enabled, X-GM-EXT-1 is advertised (both before and after auth). +func TestCapabilityGmailExtensionEnabled(t *testing.T) { + runOneToOneTest(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, _ *testSession) { + c.C("A001 Capability") + c.S(`* CAPABILITY AUTH=PLAIN ID IDLE IMAP4rev1 STARTTLS X-GM-EXT-1`) + c.S("A001 OK CAPABILITY") + + c.C(`A002 login "user" "pass"`) + c.S(`A002 OK [CAPABILITY AUTH=PLAIN ID IDLE IMAP4rev1 MOVE STARTTLS UIDPLUS UNSELECT X-GM-EXT-1] Logged in`) + + c.C("A003 Capability") + c.S(`* CAPABILITY AUTH=PLAIN ID IDLE IMAP4rev1 MOVE STARTTLS UIDPLUS UNSELECT X-GM-EXT-1`) + c.S("A003 OK CAPABILITY") + }) +} diff --git a/tests/gmail_labels_test.go b/tests/gmail_labels_test.go new file mode 100644 index 00000000..720bfe34 --- /dev/null +++ b/tests/gmail_labels_test.go @@ -0,0 +1,181 @@ +package tests + +import ( + "testing" +) + +// These integration tests exercise the X-GM-EXT-1 Gmail label extension +// end-to-end (STORE / FETCH / SEARCH) through the dummy connector, which now +// stores labels as non-exclusive label mailboxes. This is the path Paperless-NGX +// uses to tag mail over Proton Bridge IMAP. + +func TestGmailLabelsStoreFetchRoundTrip(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, _ *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + // No labels initially. + c.C(`A002 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ())`) + c.OK(`A002`) + + // Apply two labels. STORE X-GM-LABELS emits no untagged FETCH (unlike + // STORE FLAGS) — only the tagged OK. + c.C(`A003 STORE 1 +X-GM-LABELS ("Paperless" "Invoices")`) + c.OK(`A003`) + + // FETCH returns them sorted. + c.C(`A004 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ("Invoices" "Paperless"))`) + c.OK(`A004`) + + // Remove one label. + c.C(`A005 STORE 1 -X-GM-LABELS ("Invoices")`) + c.OK(`A005`) + c.C(`A006 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ("Paperless"))`) + c.OK(`A006`) + + // Remove the last label -> back to empty. + c.C(`A007 STORE 1 -X-GM-LABELS ("Paperless")`) + c.OK(`A007`) + c.C(`A008 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ())`) + c.OK(`A008`) + }) +} + +// Python's imaplib (used by Paperless-NGX) sends the label without parentheses. +// This is the exact wire form Paperless emits, so it must round-trip. +func TestGmailLabelsBareLabelForm(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, _ *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + c.C(`A002 STORE 1 +X-GM-LABELS Paperless`) + c.OK(`A002`) + + c.C(`A003 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ("Paperless"))`) + c.OK(`A003`) + }) +} + +func TestGmailLabelsWithSpaces(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, _ *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + c.C(`A002 STORE 1 +X-GM-LABELS ("Label With Spaces")`) + c.OK(`A002`) + + c.C(`A003 FETCH 1 (X-GM-LABELS)`) + c.S(`* 1 FETCH (X-GM-LABELS ("Label With Spaces"))`) + c.OK(`A003`) + }) +} + +func TestGmailLabelsMultipleMessages(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, _ *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 2@pm.me`)).expect("OK") + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 3@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + // Label a range of messages in one STORE. + c.C(`A002 STORE 1:3 +X-GM-LABELS ("Paperless")`) + c.OK(`A002`) + + c.C(`A003 FETCH 1:3 (X-GM-LABELS)`) + c.S( + `* 1 FETCH (X-GM-LABELS ("Paperless"))`, + `* 2 FETCH (X-GM-LABELS ("Paperless"))`, + `* 3 FETCH (X-GM-LABELS ("Paperless"))`, + ) + c.OK(`A003`) + }) +} + +// SEARCH X-GM-LABELS and its negation are how Paperless-NGX finds untagged mail +// (NOT X-GM-LABELS "") and verifies tagging. The connector pushes the label +// mailbox + membership asynchronously, so we flush before searching. +func TestGmailLabelsSearchRoundTrip(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t, withGmailExtension()), func(c *testConnection, s *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 2@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + // Before tagging, nothing matches and everything is "not labelled". + c.C(`A002 SEARCH X-GM-LABELS "Paperless"`) + c.S(`* SEARCH`) + c.OK(`A002`) + + // Tag only the first message. + c.C(`A003 STORE 1 +X-GM-LABELS ("Paperless")`) + c.OK(`A003`) + + // Make the connector-pushed label mailbox + membership visible. + s.flush("user") + + c.C(`A004 SEARCH X-GM-LABELS "Paperless"`) + c.S(`* SEARCH 1`) + c.OK(`A004`) + + // The dedupe query Paperless-NGX issues: everything NOT yet tagged. + c.C(`A005 SEARCH NOT X-GM-LABELS "Paperless"`) + c.S(`* SEARCH 2`) + c.OK(`A005`) + }) +} + +// With the Gmail extension disabled (the default), the X-GM-EXT-1 capability is +// not advertised and the X-GM-LABELS STORE/FETCH/SEARCH commands are rejected +// with BAD even if a non-compliant client sends them anyway. +func TestGmailLabelsRejectedWhenDisabled(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t), func(c *testConnection, _ *testSession) { + c.C("b001 CREATE saved-messages") + c.S("b001 OK CREATE") + + c.doAppend(`saved-messages`, buildRFC5322TestLiteral(`To: 1@pm.me`)).expect("OK") + + c.C(`A001 SELECT saved-messages`) + c.Se(`A001 OK [READ-WRITE] SELECT`) + + c.C(`A002 STORE 1 +X-GM-LABELS ("Paperless")`) + c.Sx(`A002 BAD`) + + c.C(`A003 FETCH 1 (X-GM-LABELS)`) + c.Sx(`A003 BAD`) + + c.C(`A004 SEARCH X-GM-LABELS "Paperless"`) + c.Sx(`A004 BAD`) + + c.C(`A005 SEARCH NOT X-GM-LABELS "Paperless"`) + c.Sx(`A005 BAD`) + }) +} diff --git a/tests/server_test.go b/tests/server_test.go index 52bf9a3f..06576617 100644 --- a/tests/server_test.go +++ b/tests/server_test.go @@ -77,6 +77,7 @@ type serverOptions struct { disableParallelism bool imapLimits limits.IMAP disableIMAPAuthenticate bool + enableGmailExtension bool reporter reporter.Reporter uidValidityGenerator imap.UIDValidityGenerator database db.ClientInterface @@ -252,6 +253,16 @@ func withDisableIMAPAuthenticate() serverOption { return &disableIMAPAuthenticateOption{} } +type gmailExtensionOption struct{} + +func (gmailExtensionOption) apply(options *serverOptions) { + options.enableGmailExtension = true +} + +func withGmailExtension() serverOption { + return &gmailExtensionOption{} +} + func defaultServerOptions(tb testing.TB, modifiers ...serverOption) *serverOptions { options := &serverOptions{ credentials: []credentials{{ @@ -334,6 +345,10 @@ func runServer(tb testing.TB, options *serverOptions, tests func(session *testSe gluonOptions = append(gluonOptions, gluon.WithDisableIMAPAuthenticate()) } + if options.enableGmailExtension { + gluonOptions = append(gluonOptions, gluon.WithGmailExtension()) + } + // Create a new gluon server. server, err := gluon.New(gluonOptions...) require.NoError(tb, err)