Skip to content

Commit 1bb3eaf

Browse files
efd6efd6
committed
o365: fix handling of empty sip IP fields and avoid script allocations
1 parent e3f7fb0 commit 1bb3eaf

File tree

5 files changed

+192
-10
lines changed

5 files changed

+192
-10
lines changed

‎packages/o365/changelog.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "2.17.2"
3+
changes:
4+
- description: Prevent convert processor failures with fields with empty string values.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/
7+
- description: Avoid script parameter allocations.
8+
type: bugfix
9+
link: https://github.com/elastic/integrations/pull/
210
- version: "2.17.1"
311
changes:
412
- description: Validate organization field type before accessing subfields.

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,42 @@
104104
"Version": 1,
105105
"Workload": "SecurityComplianceCenter"
106106
}
107+
},
108+
{
109+
"event": {
110+
"original": "{\"Category\":\"ThreatManagement\",\"UserKey\":\"SecurityComplianceAlerts\",\"Operation\":\"AlertEntityGenerated\",\"OrganizationId\":\"aaaaa14f-bbbb-cccc-dddd-eeee5a778630\",\"AlertEntityId\":\"[email protected](external, opens in a new tab or window)\",\"Source\":\"Office 365 Security & Compliance\",\"Name\":\"Email reported by user as malware or phish\",\"AlertType\":\"System\",\"RecordType\":40,\"Version\":1,\"Status\":\"Active\",\"ObjectId\":\"[email protected](external, opens in a new tab or window)\",\"ResultStatus\":\"Succeeded\",\"Comments\":\"New alert\",\"AlertLinks\":[{\"AlertLinkHref\":\"\"}],\"Data\":\"{\\\"etype\\\":\\\"User\\\",\\\"eid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"tid\\\":\\\"aaaaa14f-bbbb-cccc-dddd-eeee5a778630\\\",\\\"ts\\\":\\\"2025-05-02T05:10:44.5371861Z\\\",\\\"te\\\":\\\"2025-05-02T05:10:44.5371861Z\\\",\\\"op\\\":\\\"UserSubmission\\\",\\\"tdc\\\":\\\"1\\\",\\\"suid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"ut\\\":\\\"Regular\\\",\\\"ssic\\\":\\\"0\\\",\\\"tsd\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"sip\\\":\\\"\\\",\\\"imsgid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"srt\\\":\\\"1\\\",\\\"trc\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"ms\\\":\\\"Welkom op My company\\\",\\\"sid\\\":\\\"aaa174f-bbbb-cccc-dddd-eeeea27623b4\\\",\\\"aii\\\":\\\"aaaa109-bbb-cccc-dddd-eeeea1c1dd41\\\",\\\"md\\\":\\\"2025-05-02T10:40:16.9298292Z\\\",\\\"etps\\\":\\\"SubmissionId:aaaae50f-bbbb-4760-cccc-dddda276218e\\\",\\\"lon\\\":\\\"UserSubmission\\\"}\",\"Severity\":\"Low\",\"Workload\":\"SecurityComplianceCenter\",\"EntityType\":\"User\",\"AlertId\":\"aaaa01b-bbbb-cccc-dddd-eeeea276218e\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2025-06-03T08:10:44\",\"Id\":\"aaaabce60-bbbb-cccc-dddd-eeeea27623da\",\"UserType\":4,\"PolicyId\":\"aaaa5770-bbbb-cccc-dddd-eeee2c27bbb3\"}"
111+
},
112+
"o365audit": {
113+
"Category": "ThreatManagement",
114+
"UserKey": "SecurityComplianceAlerts",
115+
"Operation": "AlertEntityGenerated",
116+
"OrganizationId": "aaaaa14f-bbbb-cccc-dddd-eeee5a778630",
117+
"AlertEntityId": "[email protected](external, opens in a new tab or window)",
118+
"Source": "Office 365 Security & Compliance",
119+
"Name": "Email reported by user as malware or phish",
120+
"AlertType": "System",
121+
"RecordType": 40,
122+
"Version": 1,
123+
"Status": "Active",
124+
"ObjectId": "[email protected](external, opens in a new tab or window)",
125+
"ResultStatus": "Succeeded",
126+
"Comments": "New alert",
127+
"AlertLinks": [
128+
{
129+
"AlertLinkHref": ""
130+
}
131+
],
132+
"Data": "{\"etype\":\"User\",\"eid\":\"[email protected](external, opens in a new tab or window)\",\"tid\":\"aaaaa14f-bbbb-cccc-dddd-eeee5a778630\",\"ts\":\"2025-05-02T05:10:44.5371861Z\",\"te\":\"2025-05-02T05:10:44.5371861Z\",\"op\":\"UserSubmission\",\"tdc\":\"1\",\"suid\":\"[email protected](external, opens in a new tab or window)\",\"ut\":\"Regular\",\"ssic\":\"0\",\"tsd\":\"[email protected](external, opens in a new tab or window)\",\"sip\":\"\",\"imsgid\":\"[email protected](external, opens in a new tab or window)\",\"srt\":\"1\",\"trc\":\"[email protected](external, opens in a new tab or window)\",\"ms\":\"Welkom op My company\",\"sid\":\"aaa174f-bbbb-cccc-dddd-eeeea27623b4\",\"aii\":\"aaaa109-bbb-cccc-dddd-eeeea1c1dd41\",\"md\":\"2025-05-02T10:40:16.9298292Z\",\"etps\":\"SubmissionId:aaaae50f-bbbb-4760-cccc-dddda276218e\",\"lon\":\"UserSubmission\"}",
133+
"Severity": "Low",
134+
"Workload": "SecurityComplianceCenter",
135+
"EntityType": "User",
136+
"AlertId": "aaaa01b-bbbb-cccc-dddd-eeeea276218e",
137+
"UserId": "SecurityComplianceAlerts",
138+
"CreationTime": "2025-06-03T08:10:44",
139+
"Id": "aaaabce60-bbbb-cccc-dddd-eeeea27623da",
140+
"UserType": 4,
141+
"PolicyId": "aaaa5770-bbbb-cccc-dddd-eeee2c27bbb3"
142+
}
107143
}
108-
]
144+
]
109145
}

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json-expected.json

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,136 @@
259259
"user": {
260260
"id": "SecurityComplianceAlerts"
261261
}
262+
},
263+
{
264+
"@timestamp": "2025-06-03T08:10:44.000Z",
265+
"ecs": {
266+
"version": "8.11.0"
267+
},
268+
"email": {
269+
"local_id": [
270+
"aaaa109-bbb-cccc-dddd-eeeea1c1dd41"
271+
],
272+
"message_id": [
273+
"[email protected](external, opens in a new tab or window)"
274+
],
275+
"sender": {
276+
"address": [
277+
"[email protected](external, opens in a new tab or window)"
278+
]
279+
},
280+
"subject": [
281+
"Welkom op My company"
282+
],
283+
"to": {
284+
"address": [
285+
"[email protected](external, opens in a new tab or window)"
286+
]
287+
}
288+
},
289+
"event": {
290+
"action": "AlertEntityGenerated",
291+
"category": [
292+
"web"
293+
],
294+
"code": "SecurityComplianceAlerts",
295+
"id": "aaaabce60-bbbb-cccc-dddd-eeeea27623da",
296+
"kind": "alert",
297+
"original": "{\"Category\":\"ThreatManagement\",\"UserKey\":\"SecurityComplianceAlerts\",\"Operation\":\"AlertEntityGenerated\",\"OrganizationId\":\"aaaaa14f-bbbb-cccc-dddd-eeee5a778630\",\"AlertEntityId\":\"[email protected](external, opens in a new tab or window)\",\"Source\":\"Office 365 Security & Compliance\",\"Name\":\"Email reported by user as malware or phish\",\"AlertType\":\"System\",\"RecordType\":40,\"Version\":1,\"Status\":\"Active\",\"ObjectId\":\"[email protected](external, opens in a new tab or window)\",\"ResultStatus\":\"Succeeded\",\"Comments\":\"New alert\",\"AlertLinks\":[{\"AlertLinkHref\":\"\"}],\"Data\":\"{\\\"etype\\\":\\\"User\\\",\\\"eid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"tid\\\":\\\"aaaaa14f-bbbb-cccc-dddd-eeee5a778630\\\",\\\"ts\\\":\\\"2025-05-02T05:10:44.5371861Z\\\",\\\"te\\\":\\\"2025-05-02T05:10:44.5371861Z\\\",\\\"op\\\":\\\"UserSubmission\\\",\\\"tdc\\\":\\\"1\\\",\\\"suid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"ut\\\":\\\"Regular\\\",\\\"ssic\\\":\\\"0\\\",\\\"tsd\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"sip\\\":\\\"\\\",\\\"imsgid\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"srt\\\":\\\"1\\\",\\\"trc\\\":\\\"[email protected](external, opens in a new tab or window)\\\",\\\"ms\\\":\\\"Welkom op My company\\\",\\\"sid\\\":\\\"aaa174f-bbbb-cccc-dddd-eeeea27623b4\\\",\\\"aii\\\":\\\"aaaa109-bbb-cccc-dddd-eeeea1c1dd41\\\",\\\"md\\\":\\\"2025-05-02T10:40:16.9298292Z\\\",\\\"etps\\\":\\\"SubmissionId:aaaae50f-bbbb-4760-cccc-dddda276218e\\\",\\\"lon\\\":\\\"UserSubmission\\\"}\",\"Severity\":\"Low\",\"Workload\":\"SecurityComplianceCenter\",\"EntityType\":\"User\",\"AlertId\":\"aaaa01b-bbbb-cccc-dddd-eeeea276218e\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2025-06-03T08:10:44\",\"Id\":\"aaaabce60-bbbb-cccc-dddd-eeeea27623da\",\"UserType\":4,\"PolicyId\":\"aaaa5770-bbbb-cccc-dddd-eeee2c27bbb3\"}",
298+
"outcome": "success",
299+
"provider": "SecurityComplianceCenter",
300+
"type": [
301+
"info"
302+
]
303+
},
304+
"host": {
305+
"id": "aaaaa14f-bbbb-cccc-dddd-eeee5a778630"
306+
},
307+
"message": "Email reported by user as malware or phish",
308+
"o365": {
309+
"audit": {
310+
"AlertId": "aaaa01b-bbbb-cccc-dddd-eeeea276218e",
311+
"AlertType": "System",
312+
"Comments": "New alert",
313+
"CreationTime": "2025-06-03T08:10:44",
314+
"Data": {
315+
"aii": "aaaa109-bbb-cccc-dddd-eeeea1c1dd41",
316+
"eid": "[email protected](external, opens in a new tab or window)",
317+
"etps": "SubmissionId:aaaae50f-bbbb-4760-cccc-dddda276218e",
318+
"etype": "User",
319+
"flattened": {
320+
"aii": "aaaa109-bbb-cccc-dddd-eeeea1c1dd41",
321+
"eid": "[email protected](external, opens in a new tab or window)",
322+
"etps": "SubmissionId:aaaae50f-bbbb-4760-cccc-dddda276218e",
323+
"etype": "User",
324+
"imsgid": "[email protected](external, opens in a new tab or window)",
325+
"lon": "UserSubmission",
326+
"md": "2025-05-02T10:40:16.9298292Z",
327+
"ms": "Welkom op My company",
328+
"op": "UserSubmission",
329+
"sid": "aaa174f-bbbb-cccc-dddd-eeeea27623b4",
330+
"srt": "1",
331+
"ssic": "0",
332+
"suid": "[email protected](external, opens in a new tab or window)",
333+
"tdc": "1",
334+
"te": "2025-05-02T05:10:44.5371861Z",
335+
"tid": "aaaaa14f-bbbb-cccc-dddd-eeee5a778630",
336+
"trc": "[email protected](external, opens in a new tab or window)",
337+
"ts": "2025-05-02T05:10:44.5371861Z",
338+
"tsd": "[email protected](external, opens in a new tab or window)",
339+
"ut": "Regular"
340+
},
341+
"imsgid": "[email protected](external, opens in a new tab or window)",
342+
"lon": "UserSubmission",
343+
"md": "2025-05-02T10:40:16.929Z",
344+
"ms": "Welkom op My company",
345+
"op": "UserSubmission",
346+
"sid": "aaa174f-bbbb-cccc-dddd-eeeea27623b4",
347+
"srt": "1",
348+
"ssic": "0",
349+
"suid": "[email protected](external, opens in a new tab or window)",
350+
"tdc": "1",
351+
"te": "2025-05-02T05:10:44.537Z",
352+
"tid": "aaaaa14f-bbbb-cccc-dddd-eeee5a778630",
353+
"trc": "[email protected](external, opens in a new tab or window)",
354+
"ts": "2025-05-02T05:10:44.537Z",
355+
"tsd": "[email protected](external, opens in a new tab or window)",
356+
"ut": "Regular"
357+
},
358+
"ObjectId": "[email protected](external, opens in a new tab or window)",
359+
"RecordType": "40",
360+
"ResultStatus": "Succeeded",
361+
"Severity": "Low",
362+
"Source": "Office 365 Security & Compliance",
363+
"Status": "Active",
364+
"UserId": "SecurityComplianceAlerts",
365+
"UserKey": "SecurityComplianceAlerts",
366+
"UserType": "4",
367+
"Version": "1"
368+
}
369+
},
370+
"organization": {
371+
"id": "aaaaa14f-bbbb-cccc-dddd-eeee5a778630"
372+
},
373+
"related": {
374+
"user": [
375+
"[email protected](external, opens in a new tab or window)",
376+
"[email protected](external, opens in a new tab or window)"
377+
]
378+
},
379+
"rule": {
380+
"category": "ThreatManagement",
381+
"description": "[email protected](external, opens in a new tab or window)",
382+
"id": "aaaa5770-bbbb-cccc-dddd-eeee2c27bbb3",
383+
"name": "Email reported by user as malware or phish",
384+
"ruleset": "User"
385+
},
386+
"tags": [
387+
"preserve_original_event"
388+
],
389+
"user": {
390+
"id": "SecurityComplianceAlerts"
391+
}
262392
}
263393
]
264394
}

‎packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,15 +1281,18 @@ processors:
12811281
lang: painless
12821282
tag: script_known_Data
12831283
if: 'ctx.o365audit?.Data?.flattened instanceof Map'
1284-
source: >
1285-
def knownKeys = ['ad', 'af', 'aii', 'ail', 'alk', 'als', 'an', 'at',
1284+
params:
1285+
knownKeys: [
1286+
'ad', 'af', 'aii', 'ail', 'alk', 'als', 'an', 'at',
12861287
'cid', 'cpid', 'dm', 'dpn', 'eid', 'etps', 'etype', 'f3u', 'fvs',
12871288
'imsgid', 'lon', 'mat', 'md', 'ms', 'od', 'op', 'ot', 'plk', 'pud',
12881289
'reid', 'rid', 'sev', 'sict', 'sid', 'sip', 'sitmi', 'srt', 'ssic',
12891290
'suid', 'tdc', 'te', 'thn', 'tht', 'tid', 'tpid', 'tpt', 'trc', 'ts',
12901291
'tsd', 'ttdt', 'ttr', 'upfc', 'upfv', 'ut', 'von', 'wl', 'zfh', 'zfn',
1291-
'zmfh', 'zmfn', 'zu'];
1292-
for (def key : knownKeys) {
1292+
'zmfh', 'zmfn', 'zu'
1293+
]
1294+
source: >
1295+
for (def key : params.knownKeys) {
12931296
if (ctx.o365audit.Data.flattened.containsKey(key)) {
12941297
ctx.o365audit.Data[key] = ctx.o365audit.Data.flattened[key];
12951298
}
@@ -1298,6 +1301,8 @@ processors:
12981301
field: o365audit.Data.sip
12991302
type: ip
13001303
ignore_missing: true
1304+
if: ctx.o365audit?.Data?.sip != ''
1305+
ignore_failure: true
13011306
- date:
13021307
field: o365audit.Data.at
13031308
target_field: o365audit.Data.at
@@ -1389,15 +1394,18 @@ processors:
13891394
lang: painless
13901395
tag: script_known_Data.Entities
13911396
if: ctx.o365audit?.Data?.flattened?.Entities instanceof List
1397+
params:
1398+
knownEntityKeys: [
1399+
'InternetMessageId', 'NetworkMessageId', 'OriginalDeliveryLocation',
1400+
'P1Sender', 'P2Sender', 'PhishConfidenceLevel', 'Recipient', 'SenderIP', 'Subject',
1401+
'ThreatDetectionMethods', 'Upn'
1402+
]
13921403
source: >
13931404
ctx._tmp = [:];
13941405
ctx._tmp.entities = [:];
1395-
def knownEntityKeys = ['InternetMessageId', 'NetworkMessageId', 'OriginalDeliveryLocation',
1396-
'P1Sender', 'P2Sender', 'PhishConfidenceLevel', 'Recipient', 'SenderIP', 'Subject',
1397-
'ThreatDetectionMethods', 'Upn'];
13981406
for (def entity: ctx.o365audit.Data.flattened.Entities) {
13991407
if (entity instanceof Map) {
1400-
for (def key : knownEntityKeys) {
1408+
for (def key : params.knownEntityKeys) {
14011409
if (! ctx._tmp.entities.containsKey(key)) {
14021410
ctx._tmp.entities[key] = [];
14031411
}

‎packages/o365/manifest.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: o365
22
title: Microsoft Office 365
3-
version: "2.17.1"
3+
version: "2.17.2"
44
description: Collect logs from Microsoft Office 365 with Elastic Agent.
55
type: integration
66
format_version: "3.2.3"

0 commit comments

Comments
 (0)