From f072d12d10b6e4f4759f34336440a22f4dbc0bb0 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 10:30:30 -0400 Subject: [PATCH 01/20] Update security_alert_overview.xml Updated base search to remove outdated index reference without macro. --- .../default/data/ui/views/security_alert_overview.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/github_app_for_splunk/default/data/ui/views/security_alert_overview.xml b/github_app_for_splunk/default/data/ui/views/security_alert_overview.xml index a85551d..f76d40b 100644 --- a/github_app_for_splunk/default/data/ui/views/security_alert_overview.xml +++ b/github_app_for_splunk/default/data/ui/views/security_alert_overview.xml @@ -2,14 +2,14 @@ Advanced Security Overview - index=gh_vuln OR (`github_webhooks` alert.created_at=*) - | eval type=case((eventtype="GitHub::CodeScanning"), "Code Scanning Alert", (eventtype="GitHub::VulnerabilityAlert"), "Dependabot Alert", (eventtype="GitHub::SecretScanning"), "Secret Scanning Alert") - | eval url=case((eventtype="GitHub::CodeScanning"), 'alert.html_url', (eventtype="GitHub::VulnerabilityAlert"), 'repository.html_url'+"/security/dependabot/"+'alert.number', (eventtype="GitHub::SecretScanning"), 'alert.html_url') + `github_webhooks` alert.created_at=* + | eval type=case((eventtype="GitHub::CodeScanning"), "Code Scanning Alert", (eventtype="GitHub::VulnerabilityAlert"), "Dependabot Alert", (eventtype="GitHub::SecretScanning"), "Secret Scanning Alert") + | eval url=case((eventtype="GitHub::CodeScanning"), 'alert.html_url', (eventtype="GitHub::VulnerabilityAlert"), 'repository.html_url'+"/security/dependabot/"+'alert.number', (eventtype="GitHub::SecretScanning"), 'alert.html_url') | eval reason=case((type="Dependabot Alert"),'alert.affected_package_name',(type="Code Scanning Alert"), 'alert.rule.name', (type="Secret Scanning Alert"), 'alert.secret_type'), id=case((type="Dependabot Alert"),'alert.external_identifier',(type="Code Scanning Alert"), 'alert.rule.id', (type="Secret Scanning Alert"), 'alert.number'), severity=case((type="Dependabot Alert"),'alert.severity',(type="Code Scanning Alert"), 'alert.rule.security_severity_level', (type="Secret Scanning Alert"), "high"), repository = 'repository.full_name' | stats latest(action) as status, earliest(alert.created_at) as created_at, latest(alert.number) as number by repository, reason, id, type, severity, url | eval source=type - | eval age = toString(round(now() - strptime(created_at, "%Y-%m-%dT%H:%M:%S")),"Duration") - | search severity IN("*") status IN("*") type IN("*") + | eval age = toString(round(now() - strptime(created_at, "%Y-%m-%dT%H:%M:%S")),"Duration") + | search severity IN("*") status IN("*") type IN("*") | sort -age $timeTkn.earliest$ From f5b689bc6eb7ae788b9d071d0ab7512e4c3c46f3 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 10:38:54 -0400 Subject: [PATCH 02/20] Fix for Workflow Analysis Repo Selector Updated the workflow analysis dashboard and associated javascript to fix the broken repo select input and mismatched time token. --- .../appserver/static/workflowdetails.js | 4 ++-- .../default/data/ui/views/workflow_analysis.xml | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/github_app_for_splunk/appserver/static/workflowdetails.js b/github_app_for_splunk/appserver/static/workflowdetails.js index dbf8f8f..e7b23bc 100644 --- a/github_app_for_splunk/appserver/static/workflowdetails.js +++ b/github_app_for_splunk/appserver/static/workflowdetails.js @@ -17,8 +17,8 @@ require([ preview: true, cache: true, search: mvc.tokenSafe("index=github_webhook eventtype=\"GitHub::Workflow\" \"workflow_job.run_id\"=$workflow_id$| fields * | eval queued=if(action==\"queued\",_time,null), started=if(action==\"in_progress\",_time,null), completed=if(action==\"completed\",_time,null) | stats latest(workflow_job.conclusion) as status, latest(workflow_job.name) as Name, latest(queued) as queued, latest(started) as started, latest(completed) as completed by workflow_job.id | eval queueTime=toString(round(started-queued),\"Duration\"), runTime=toString(round(completed-started),\"Duration\"), totalTime=toString(round(completed-queued),\"Duration\"), status=if(status==\"null\",\"in_progress\",status) | rename workflow_job.id AS JobID | fields status, Name, JobID, queueTime, runTime, totalTime"), - earliest_time: mvc.tokenSafe("$field1.earliest$"), - latest_time: mvc.tokenSafe("$field1.latest$") + earliest_time: mvc.tokenSafe("timeTkn.earliest$"), + latest_time: mvc.tokenSafe("timeTkn.latest$") }); // Create a table for a custom row expander diff --git a/github_app_for_splunk/default/data/ui/views/workflow_analysis.xml b/github_app_for_splunk/default/data/ui/views/workflow_analysis.xml index 887da28..2638fe0 100644 --- a/github_app_for_splunk/default/data/ui/views/workflow_analysis.xml +++ b/github_app_for_splunk/default/data/ui/views/workflow_analysis.xml @@ -1,7 +1,7 @@

Workflow Analysis

- + -24h@h @@ -10,6 +10,13 @@ Repositories + repository.name + repository.name + + `github_webhooks` eventtype="GitHub::Workflow"|dedup repository.name| table repository.name + $timeTkn.earliest$ + $timeTkn.latest$ + All * * @@ -21,8 +28,8 @@ Workflow Conclusions Over Time `github_webhooks` "workflow_run.name"="*" | spath "repository.full_name" | search repository.full_name="$repos$" | stats latest(_time) as _time, latest(workflow_run.conclusion) as workflow_run.conclusion by repository.full_name,workflow_run.name,workflow_run.id | timechart count by workflow_run.conclusion span=1h | rename null as "in-progress" - $field1.earliest$ - $field1.latest$ + $timeTkn.earliest$ + $timeTkn.latest$ 1 From ad2c4b274faa96af75a2ae07cf3d0c078f12db32 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 10:39:44 -0400 Subject: [PATCH 03/20] Update workflowdetails.js Replace hard coded index with webhooks macro. --- github_app_for_splunk/appserver/static/workflowdetails.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_app_for_splunk/appserver/static/workflowdetails.js b/github_app_for_splunk/appserver/static/workflowdetails.js index e7b23bc..d9498f4 100644 --- a/github_app_for_splunk/appserver/static/workflowdetails.js +++ b/github_app_for_splunk/appserver/static/workflowdetails.js @@ -16,7 +16,7 @@ require([ id: "workflow_details", preview: true, cache: true, - search: mvc.tokenSafe("index=github_webhook eventtype=\"GitHub::Workflow\" \"workflow_job.run_id\"=$workflow_id$| fields * | eval queued=if(action==\"queued\",_time,null), started=if(action==\"in_progress\",_time,null), completed=if(action==\"completed\",_time,null) | stats latest(workflow_job.conclusion) as status, latest(workflow_job.name) as Name, latest(queued) as queued, latest(started) as started, latest(completed) as completed by workflow_job.id | eval queueTime=toString(round(started-queued),\"Duration\"), runTime=toString(round(completed-started),\"Duration\"), totalTime=toString(round(completed-queued),\"Duration\"), status=if(status==\"null\",\"in_progress\",status) | rename workflow_job.id AS JobID | fields status, Name, JobID, queueTime, runTime, totalTime"), + search: mvc.tokenSafe("`github_webhooks` eventtype=\"GitHub::Workflow\" \"workflow_job.run_id\"=$workflow_id$| fields * | eval queued=if(action==\"queued\",_time,null), started=if(action==\"in_progress\",_time,null), completed=if(action==\"completed\",_time,null) | stats latest(workflow_job.conclusion) as status, latest(workflow_job.name) as Name, latest(queued) as queued, latest(started) as started, latest(completed) as completed by workflow_job.id | eval queueTime=toString(round(started-queued),\"Duration\"), runTime=toString(round(completed-started),\"Duration\"), totalTime=toString(round(completed-queued),\"Duration\"), status=if(status==\"null\",\"in_progress\",status) | rename workflow_job.id AS JobID | fields status, Name, JobID, queueTime, runTime, totalTime"), earliest_time: mvc.tokenSafe("timeTkn.earliest$"), latest_time: mvc.tokenSafe("timeTkn.latest$") }); @@ -96,7 +96,7 @@ require([ return cell.field === 'Run ID'; }); - this._searchManager.set({ search: 'index=github_webhook (workflow_run.id='+workflowIDCell.value+' OR workflow_job.run_id='+workflowIDCell.value+') | eval started=if(action=="requested", _time, null), completed=if(action=="completed", _time,null) | stats latest(workflow_run.conclusion) as Status, earliest(started) as Started, latest(completed) as Completed, latest(workflow_run.head_branch) as Branch, latest(workflow_run.event) as Trigger | eval Duration=tostring(Completed-Started, "Duration") | fields Status, Duration, Branch, Trigger | eval Details="Click here for Workflow Details" | transpose|rename column AS Details| rename "row 1" AS values'}); + this._searchManager.set({ search: '`github_webhooks` (workflow_run.id='+workflowIDCell.value+' OR workflow_job.run_id='+workflowIDCell.value+') | eval started=if(action=="requested", _time, null), completed=if(action=="completed", _time,null) | stats latest(workflow_run.conclusion) as Status, earliest(started) as Started, latest(completed) as Completed, latest(workflow_run.head_branch) as Branch, latest(workflow_run.event) as Trigger | eval Duration=tostring(Completed-Started, "Duration") | fields Status, Duration, Branch, Trigger | eval Details="Click here for Workflow Details" | transpose|rename column AS Details| rename "row 1" AS values'}); // $container is the jquery object where we can put out content. // In this case we will render our chart and add it to the $container $container.append(this._TableView.render().el); From 143e1896bdccc3a7fb882a0a1f6a2648bb791697 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 10:45:01 -0400 Subject: [PATCH 04/20] Update workflow_analytics.xml Change repo selector time selector to use token instead of hard coded 30 days. --- .../default/data/ui/views/workflow_analytics.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_app_for_splunk/default/data/ui/views/workflow_analytics.xml b/github_app_for_splunk/default/data/ui/views/workflow_analytics.xml index 59d5c8d..e6ea68c 100644 --- a/github_app_for_splunk/default/data/ui/views/workflow_analytics.xml +++ b/github_app_for_splunk/default/data/ui/views/workflow_analytics.xml @@ -14,8 +14,8 @@ repository.name `github_webhooks` eventtype="GitHub::Workflow"|dedup repository.name| table repository.name - -30d@d - now + $timeTkn.earliest$ + $timeTkn.latest$ All * From e1664998698ceb5e227536bff11598237cb8c2de Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 10:47:54 -0400 Subject: [PATCH 05/20] Added workflow logs macro Remove hard coded reference to a specific index for workflow logs and use a macro. --- .../default/data/ui/views/workflow_details.xml | 2 +- github_app_for_splunk/default/macros.conf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/github_app_for_splunk/default/data/ui/views/workflow_details.xml b/github_app_for_splunk/default/data/ui/views/workflow_details.xml index 2662cbf..846a3a3 100644 --- a/github_app_for_splunk/default/data/ui/views/workflow_details.xml +++ b/github_app_for_splunk/default/data/ui/views/workflow_details.xml @@ -194,7 +194,7 @@ Workflow Run Logs - index="github_workflow_logs" workflowID::$workflow_id$ | sort _time + `github_workflow_logs` workflowID::$workflow_id$ | sort _time 0 1 diff --git a/github_app_for_splunk/default/macros.conf b/github_app_for_splunk/default/macros.conf index 8b8357a..d8b3480 100644 --- a/github_app_for_splunk/default/macros.conf +++ b/github_app_for_splunk/default/macros.conf @@ -11,3 +11,6 @@ iseval = 0 definition = index=github_webhook iseval = 0 +[github_workflow_logs] +definition = index="github_workflow_logs" +iseval = 0 From f70329bfc15a348e0242411be4363efa40023f1e Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 11:19:44 -0400 Subject: [PATCH 06/20] Update Docs Updating Docs to refer to official Splunk Add-on for GitHub. Removing built in documentation in favor of official docs and references to GitHub repo documentation. --- docs/ghe_audit_logs.MD | 61 +---- docs/ghes_syslog_setup.MD | 2 +- .../default/data/ui/views/api_config.xml | 232 ------------------ .../default/data/ui/views/webhook_config.xml | 77 ------ .../default/data/ui/views/welcome_page.xml | 6 +- 5 files changed, 5 insertions(+), 373 deletions(-) delete mode 100644 github_app_for_splunk/default/data/ui/views/api_config.xml delete mode 100644 github_app_for_splunk/default/data/ui/views/webhook_config.xml diff --git a/docs/ghe_audit_logs.MD b/docs/ghe_audit_logs.MD index a0fee5b..7be7d2c 100644 --- a/docs/ghe_audit_logs.MD +++ b/docs/ghe_audit_logs.MD @@ -1,13 +1,5 @@ # GitHub Enterprise Audit Log Monitoring -> Splunk modular input plugin to fetch the enterprise audit log from GitHub Enterprise - -Support for modular inputs in Splunk Enterprise 5.0 and later enables you to add new types of inputs to Splunk Enterprise that are treated as native Splunk Enterprise inputs. - -This modular input makes an HTTPS request to the GitHub Enterprise's Audit Log REST API endpoint at a definable interval to fetch audit log data. - -![Splunk modular input demo](./images/C70F5295-D9FA-48FC-90CA-A7BD397AEC35.png) - ## Prerequisites - Splunk v8.x+ @@ -15,31 +7,7 @@ This modular input makes an HTTPS request to the GitHub Enterprise's Audit Log R ## Installation -1. SSH to your Splunk server - -2. Download the latest release from [Releases](https://github.com/splunk/github-audit-log-monitoring-add-on-for-splunk/releases) - -3. Copy the tarball to the apps directory and extract it: - - ```sh - $ cp splunk-ghe-audit-log-monitoring-.tar.gz $SPLUNK_HOME/etc/apps/ - - $ mkdir -p $SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring - - $ tar xf $SPLUNK_HOME/etc/apps/splunk-ghe-audit-log-monitoring-.tar.gz -C $SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring --strip-components=1 - - # Optional depending on the user executing the previous actions - $ sudo chown -R splunk:splunk $SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring - - # Make the state directory writable by the group - $ sudo chmod -R 775 /opt/splunk/etc/apps/ghe_audit_log_monitoring/state - ``` - -4. Restart the Splunk server - -5. Generate a Personal Access Token in GitHub.com (PAT must be generated by an Enterprise Owner). - -6. Configure and the GitHub Enterprise Audit Log Monitoring by entering the necessary information in the input fields +Installation and configuration documents for the (Splunk Add-on for GitHub)[https://docs.splunk.com/Documentation/AddOns/released/GitHub/About] is available in our official Splunk docs. This add-on can be used for both GitHub Enterprise Cloud and Server. To configure for each specific environment, please refer to the official docs. ## Configuration @@ -155,30 +123,3 @@ If you've enabled debug mode be ready to change your personal access token becau ### Why can't I use a GitHub app instead of a personal access token? GitHub apps cannot be installed on the enterprise level. The REST API requires enterprise admin privileges which are out of scope for GitHub apps. - -## Troubleshooting - -### Read logs in Splunk - -You can use this search query to fetch all the logs belonging to this module when **Debug Mode** is enabled. - -```sh -index="_internal" source="/opt/splunk/var/log/splunk/splunkd.log" ghe_audit_log_monitoring -``` - -### Test the modular input for syntax problems - -Run this test if you don't see anything in the logs (which is a highly unlikely scenario). This will display any syntax errors if there are any. - -```sh -sudo $SPLUNK_HOME/bin/splunk cmd python $SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring/bin/ghe_audit_log_monitoring.py -``` - -### Where are state files stored? - -State files for enterprises are stored in this directory: - -```sh -$SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring/state/ -``` -Test diff --git a/docs/ghes_syslog_setup.MD b/docs/ghes_syslog_setup.MD index 7e32826..40de445 100644 --- a/docs/ghes_syslog_setup.MD +++ b/docs/ghes_syslog_setup.MD @@ -1,3 +1,3 @@ # Sending GitHub Enterprise Server Logs to Splunk -GitHub Enterprise Server comes with syslog-ng built in to send data to platforms like Splunk: https://docs.github.com/en/enterprise-server@3.3/admin/user-management/monitoring-activity-in-your-enterprise/log-forwarding. Following those directions will allow you to easily onboard logs to Splunk. To take advantage of GitHub Enterprise Server's built in syslog, you can direct GHES to a Splunk Connect for Syslog endpoint which has built in capability to parse GitHub Enterprise Server logs. Pairing that with the [Splunk Add-On for GitHub](https://splunkbase.splunk.com/app/6254/) will enable proper field extractions and field aliases. +GitHub Enterprise Server comes with syslog-ng built in to send data to platforms like Splunk and we can take advantage of that with the (Splunk Add-on for GitHub)[https://splunkbase.splunk.com/app/6254/]. Setup details and documentation is available on (Splunk Docs)[https://docs.splunk.com/Documentation/AddOns/released/GitHub/About]. diff --git a/github_app_for_splunk/default/data/ui/views/api_config.xml b/github_app_for_splunk/default/data/ui/views/api_config.xml deleted file mode 100644 index 6e20b1f..0000000 --- a/github_app_for_splunk/default/data/ui/views/api_config.xml +++ /dev/null @@ -1,232 +0,0 @@ - - Audit Log Config - - - -

GitHub Enterprise Audit Log Monitoring

This modular input makes an HTTPS request to the GitHub Enterprise's Audit Log REST API endpoint at a definable interval to fetch audit log data.

Prerequisites

Splunk v8+

Installation

-
Download the latest release of the Splunk Add-On for GitHub Enterprise Audit Logs from SplunkBase
-
-
Go to Apps > Manage Apps in the toolbar menu.
-
-
Use the "Install app from file" button to upload the spl file you downloaded from Splunkbase
-
-
Generate a Personal Access Token in GitHub Enterprise with the site_admin scope.
-
-
Under Settings > Data inputs, there should be a new option called GitHub Audit Log Monitoring, click "+ Add new"
-
-
Configure the Input by entering the necessary information in the input fields. Don't forget to define the Index for the data to be stored in. This option is under the "More settings" option.
-
-
Under Settings > Advanced Search, select Search Macros. You'll need to update the github_source macro to use the Index you assigned above.
-

Configuration

Personal Access Token Scope

The following are the required scopes for the personal access token allowing the module to fetch the audit log entries successfully:

[x] admin:enterprise Full control of enterprises -
- [x] manage_billing:enterprise Read and write enterprise billing data -
- [x] read:enterprise Read enterprise profile data -
-

Input Fields

-
- name -
-
- This is name of your instance. You can have multiple modular inputs running simultaneously. However, this is not a recommended behavior for this module.
- Takes: alpha-numeric, white spaces and symbol characters
- Example: GHE-enterprise-name -
-
-
- Hostname -
-
- This is the hostname of your GitHub Enterprise instance. Make sure there are no trailing / in the URL provided. This could either be a FQDN or an IP address. Do not append any paths beyond the tld.
- Example: https://api.github.com -
-
-
- Enterprise -
-
- The enterprise name for which to fetch audit log events
-
-
- Personal Access Token -
-
- This is your personal access token that you generate for your or a service account in GitHub Enterprise. This module requires that the personal access token be created with the site_admin scope. This is a very sensitive token so make sure to keep it secure at all times!
- Security: The personal access token is encrypted and stored in Splunk's password storage. After you configure it the first time it will be replaced in Splunk's UI with a unique identifier. This identifier will be used by the module to fetch the personal access token before making the API request to GitHub Enterprise.
- Takes: a 40 character token
- Example: d0e117b6ad471der3rjdowcc401a95d09202119f -
-
-
- Event Types -
-
- The audit log contains multiple event types. This field allows you to specify which events to include:
  - web - returns web (non-Git) events
  - git - returns Git events
  - all - returns both web and Git events
  -
- - More details -
-
-
- Maximum Entries Per Run -
-
- The maximum number of events / entries to fetch each time the script runs. To understand how to calculate the maximum number of entries and interval to best fit your organization go to the Tweaking throughput section below.
-
-
- Verify Self-Signed Certificates -
-
- This is a parameter passed to the get() method in the Requests library. If the checkbox is cheked then the SSL certificate will be verified like a browser does and Requests will throw a SSLError if it’s unable to verify the certificate. Uncheck this box if you are using self-signed certificates.
-
-
- Debug Mode -
-
- The personal access token will be leaked in the splunkd logs. DO NOT ENABLE unless you are ready to update your personal access token.
- If you are experiencing issues and the module is not operating as intended, you can enable this mode to seethe module's debugging information in the splunkd logs.
-
-
- Interval -
-
- Takes a cron expression as defined in the Splunk docs.
- Example: 30 * * * * -
  - At minute 30 of every hour. For example, if you set this CRON job at 11:02, your job will begin running at 11:30, 12:30, 1:30, etc...
  -
- Example: */5 * * * * -
  - Every 5 minutes
  -
- Example: 300 -
  - Every 300 seconds or 5 minutes
  -
-

Tweaking throughput

This modular input fetches events by calling the Enterprise Audit Log API. This API returns a maximum of 100 events / entries per page. The pagination algorithm can fetch events up to the maximum entries per run defined. It's important to tweak the maximum entries per run and interval parameters to have the ability to fetch your data in a timely manner and stay as close to real-time as possible.

- Example: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Enterprise Events per minute Maximum entries per run Interval API calls used Guidance
Evil-Corp 1000 1000 */1 * * * * 3000 per hour The modular input should be able to handle this with ease.
Poizen-Inc 5000 5000 */1 * * * * 600 per hour We are approaching API rate limit per hour. Depending on latency, 5000 entries = 50 API calls per minute. One minute might not be sufficient to fetch all this data.
Monsters-Inc 10000 2000 */1 * * * * 1200 per hour We are approaching API rate limit per hour. Depending on latency, 5000 entries = 50 API calls per minute. One minute might not be sufficient to fetch all this data.
-

Enterprise	Events per minute	Maximum entries per run	Interval	API calls used	Guidance
Evil-Corp	1000	1000	/1 * * *	3000 per hour	The modular input should be able to handle this with ease.
Poizen-Inc	5000	5000	/1 * * *	600 per hour	We are approaching API rate limit per hour. Depending on latency, 5000 entries = 50 API calls per minute. One minute might not be sufficient to fetch all this data.
Monsters-Inc	10000	2000	/1 * * *	1200 per hour	We are approaching API rate limit per hour. Depending on latency, 5000 entries = 50 API calls per minute. One minute might not be sufficient to fetch all this data.

FAQs

How is my Personal Access Token secured?

On the first run the modular input will identify that your personal access token (PAT) is not encrypted. It will encrypt your PAT and store it in Splunk's credentials manager. It will replace the plaintext PAT with an md5 hash of an identifying key.

Your personal access token is only visible in plaintext from the time you configure the modular input instance until the first run.

Does the interval field access only cron syntax?

No, you can enter the number of seconds instead.

I enabled debug mode, what now?

If you've enabled debug mode be ready to change your personal access token because it will most likely be leaked into the Splunk logs in plain text.

Why can't I use a GitHub app instead of a personal access token?

GitHub apps cannot be installed on the enterprise level. The REST API requires enterprise admin privileges which are out of scope for GitHub apps.

Troubleshooting

Read logs in Splunk

You can use this search query to fetch all the logs belonging to this module when Debug Mode is enabled.

-          
-            index="_internal" source="/opt/splunk/var/log/splunk/splunkd.log" ghe_audit_log_monitoring
-
-

Test the modular input for syntax problems

Run this test if you don't see anything in the logs (which is a highly unlikely scenario). This will display any syntax errors if there are any.

-          sudo $SPLUNK_HOME
-            /bin/splunk cmd python $SPLUNK_HOME
-            /etc/apps/ghe_audit_log_monitoring/bin/ghe_audit_log_monitoring.py
-
-

Where are state files stored?

State files for enterprises are stored in this directory:

-          
-            $SPLUNK_HOME/etc/apps/ghe_audit_log_monitoring/state/
-
-

- - - - diff --git a/github_app_for_splunk/default/data/ui/views/webhook_config.xml b/github_app_for_splunk/default/data/ui/views/webhook_config.xml deleted file mode 100644 index 4a41155..0000000 --- a/github_app_for_splunk/default/data/ui/views/webhook_config.xml +++ /dev/null @@ -1,77 +0,0 @@ - - Webhook Config - - - -

Using GitHub Webhooks

GitHub Webhooks are a great way to collect rich information as it occurs. You can easily enable webhooks within the GitHub UI and can even select specific actions on which to trigger a webhook call to Splunk. This is only available at the Organization level and will require this to be done for each Org as desired. To do so, you'll need to configure Splunk as a receiver and then setup the webhooks within GitHub.

Configuring Splunk to receive Webhooks

Splunk's HTTP Event Collector (HEC) is a quick and easy endpoint built to receive data from other producers like GitHub.

Steps -

Under Settings > Data Inputs, click HTTP Event Collector
Assuming HEC is enabled, click the New Token button
You can provide any name you want, however it is recommended to use something that will easily identify it like github_webhooks or similar based on your company's naming conventions, if they exist.
Unless required by your SPlunk administrator, the rest of this page can be left as is and continue onto the next step.
You'll want to click select for Source Type, and a new selection box will appear below that.
Under the Application option, there should be an entry for github_json, however you may need to use the little search bar to find it.
For App Context, you'll want to select Splunk App for GitHub
Next select the index created for this data. If none exist, create a new Index. Names like github or the like are recommended, depending on corporate naming conventions.
Lastly, click the Review button and confirm the data is correct and hit Submit.

Your token is now available to collect data, however we'll need to enable that token to allow Query String Authentication using that token. For this, you'll need command line access to your Splunk environment or be using a deployment server to deploy apps to Splunk.

To enable Query String Authentication, you'll need to update the inputs.conf file within the Splunk App for GitHub local directory. In that file, there will be a stanza with the name and value of the token you created. At the end of that stanza, you'll need to add allowQueryStringAuth = true and then restart Splunk. This is best done with the help of your Splunk team, so please reach out to them for assistance on this step.

Setting Up GitHub Webhooks

Webhooks are a simple push mechanism that will send an event each time the webhook is triggered. Unfortunately, Webhooks are unique to each Organization and will need to be setup for each Org as desired. To do this, a user will need to be an Admin for the Org.

Steps

In your Organization Settings page, select Webhooks from the menu on the left.
On this page, you'll see all the existing Webhooks, click the Add webhook button to add one to send data to Splunk
The Payload URL will be the Splunk HTTP Event Collector endpoint that was enabled above. It should look something like: https://YOUR SPLUNK URL:8088/services/collector/raw?token=THE TOKEN FROM ABOVE. The port here of 8088 may be different for your Splunk Environment, so please confirm the HEC port with your Splunk Admin team.
For Content Type, you'll want to select application/json as the best option.
You can choose to send just push events, All events, or manually select specific events from the list available. However, only some events have related Splunk eventtypes available to differentiate them within Splunk. See the table of available eventtypes below.
Once you click Add Webhook, a sample event will be triggered and it's status and response from the HTTP Event Collector should show below. Confirm that the response is OK. Otherwise triage as needed based on the HTTP Response provided.

Once that is complete and webhooks are triggering, you'll want to update the macro used for Webhook based dashboards. To do this:

In Splunk, under Settings > Advanced Search, you'll see an entry for Macros, click that.
There is a macro called github_webhooks, you'll need to update it to specificy the Index used by the HTTP Event Collector token created earlier. Once saved, any dashboards that report on Webhook events should automatically start displaying data.

Available Webhook Eventtypes

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Splunk Eventtype	GitHub Webhook Event	Description
GitHub::Repo	Repositories	Repository created, deleted, archived, unarchived, publicized, privatized, edited, renamed, or transferred.
GitHub::Push	Pushes	Git push to a repository.
GitHub::PullRequest	Pull requests	Pull request opened, closed, reopened, edited, assigned, unassigned, review requested, review request removed, labeled, unlabeled, synchronized, ready for review, converted to draft, locked, unlocked, auto merge enabled, auto merge disabled, milestoned, or demilestoned.
GitHub::PullRequest::Review	Pull request reviews	Pull request review submitted, edited, or dismissed.
GitHub::CodeScanning	Code Scanning alerts	Alert created, fixed, reopened, appeared in branch, closed by user, or reopened by user.

- - - - diff --git a/github_app_for_splunk/default/data/ui/views/welcome_page.xml b/github_app_for_splunk/default/data/ui/views/welcome_page.xml index 70d7d04..84dcbe6 100644 --- a/github_app_for_splunk/default/data/ui/views/welcome_page.xml +++ b/github_app_for_splunk/default/data/ui/views/welcome_page.xml @@ -85,10 +85,10 @@

How to collect GitHub Data

- GitHub has several ways to collect data from their services depending on your needs. Information is available within the App on how to collect different types of data from GitHub: + GitHub has several ways to collect data from their services depending on your needs.

Audit Log data is available through a Splunk Add-On
Rich commit, pull request, and Code Scanning data is available through GitHub Webhooks
Audit Log data is available through a Splunk Add-On
Rich commit, pull request, and Code Scanning data is available through GitHub Webhooks

From 20df6c319dd6ec3978dd90e6c1588406f4090f39 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 11:33:21 -0400 Subject: [PATCH 07/20] Eventtype Update Updated Eventtype list and added tags.conf to tag eventtypes for data model use in the future. --- github_app_for_splunk/default/eventtypes.conf | 46 ++++++++++++------- github_app_for_splunk/default/tags.conf | 20 ++++++++ 2 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 github_app_for_splunk/default/tags.conf diff --git a/github_app_for_splunk/default/eventtypes.conf b/github_app_for_splunk/default/eventtypes.conf index 5ef01a1..acdd685 100644 --- a/github_app_for_splunk/default/eventtypes.conf +++ b/github_app_for_splunk/default/eventtypes.conf @@ -1,9 +1,27 @@ +[GitHub::Change] +search = `github_source` action=* sourcetype="github:enterprise:audit" OR sourcetype="github_audit" + +[GitHub::CodeScanning] +search = `github_webhooks` action IN ("appeared_in_branch", "closed_by_user", "created", "fixed", "reopened", "reopened_by_user") "alert.created_at"=* + +[GitHub::CodeVulnerability] +search = `github_webhooks` (eventtype="GitHub::CodeScanning") "alert.html_url"="*/security/code-scanning/*" + [GitHub::Issue] search = `github_webhooks` action IN ("opened","edited","deleted","pinned","unpinned","closed","reopened","assigned","unassigned","labeled","unlabeled","locked","unlocked","transferred","milestoned","demilestoned") "issue.number"=* NOT "comment.body"=* [GitHub::Issue::Comment] search = `github_webhooks` action IN ("created","edited","deleted") "issue.number"=* "comment.body"=* +[GitHub::Project] +search = `github_webhooks` action IN ("created","edited","closed","reopenend","deleted") "project.number"=* + +[GitHub::Project::Card] +search = `github_webhooks` action IN ("created","edited","moved","converted","deleted") "project_card.id"=* + +[GitHub::Project::Column] +search = `github_webhooks` action IN ("created","edited","moved","deleted") "project_column.id"=* + [GitHub::PullRequest] search = `github_webhooks` action IN ("opened","edited","closed","assigned","unassigned","review_requested","review_request_removed","ready_for_review","converted_to_draft","labeled","unlabeled","synchronize","auto_merge_enabled","auto_merge_disabled","locked","unlocked","reopened") number=* "pull_request.id"=* @@ -13,23 +31,14 @@ search = `github_webhooks` action IN ("submitted","edited","dismissed") pull_req [GitHub::Push] search = `github_webhooks` after=* before=* "commits{}.id"=* ref=* "pusher.name"=* -[GitHub::Repo] -search = `github_webhooks` action IN ("created","deleted","archived","unarchived","edited","renamed","transferred","publicized","privatized") "repository.name"=* NOT "pull_request.id"=* NOT "project_card.id"=* NOT "project.number"=* NOT "project_column.id"=* NOT "check_run.id"=* NOT "alert.created_at"=* NOT "alert.number"=* - -[GitHub::Project] -search = `github_webhooks` action IN ("created","edited","closed","reopenend","deleted") "project.number"=* - -[GitHub::Project::Card] -search = `github_webhooks` action IN ("created","edited","moved","converted","deleted") "project_card.id"=* - -[GitHub::Project::Column] -search = `github_webhooks` action IN ("created","edited","moved","deleted") "project_column.id"=* +[GitHub::Release] +search = `github_webhooks` action IN ("released","published") release.id=* -[GitHub::Workflow] -search = `github_webhooks` action IN ("queued","created","in_progress","completed") workflow_job.id=* +[GitHub::Release::Push] +search = `github_webhooks` after=* before=* ref=refs/tags* -[GitHub::CodeScanning] -search = `github_webhooks` action IN ("appeared_in_branch", "closed_by_user", "created", "fixed", "reopened", "reopened_by_user") "alert.created_at"=* +[GitHub::Repo] +search = `github_webhooks` action IN ("created","deleted","archived","unarchived","edited","renamed","transferred","publicized","privatized") "repository.name"=* NOT "pull_request.id"=* NOT "project_card.id"=* NOT "project.number"=* NOT "project_column.id"=* NOT "check_run.id"=* NOT "alert.created_at"=* NOT "alert.number"=* [GitHub::SecretScanning] search = `github_webhooks` action IN ("created", "resolved") "alert.secret_type"=* @@ -37,5 +46,8 @@ search = `github_webhooks` action IN ("created", "resolved") "alert.secret_type" [GitHub::VulnerabilityAlert] search = `github_webhooks` action IN ("create", "dismiss", "resolve") "alert.external_identifier"=* -[GitHub::Release] -search = `github_webhooks` action IN ("released","published") release.id=* +[GitHub::Workflow] +search = `github_webhooks` action IN ("queued","created","in_progress","completed") workflow_job.id=* + +[github:enterprise:authentication] +search = `github_source` sourcetype=GithubEnterpriseServerAuditLog app=* authentication_service=* signature=* diff --git a/github_app_for_splunk/default/tags.conf b/github_app_for_splunk/default/tags.conf new file mode 100644 index 0000000..1eece52 --- /dev/null +++ b/github_app_for_splunk/default/tags.conf @@ -0,0 +1,20 @@ +[sourcetype =%20github_audit] + +[sourcetype=github_audit] +audit = enabled +change = enabled + +[eventtype=GitHub%3A%3AVulnerabilityAlert] +report = enabled +vulnerability = enabled + +[eventtype=GitHub%3A%3AChange] +change = enabled +audit = enabled + +[eventtype=GitHub%3A%3ACodeVulnerability] +report = enabled +vulnerability = enabled + +[eventtype=github%3Aenterprise%3Aauthentication] +authentication = enabled From 790c89307809ffd2ffbe6cf86622915a6ca1fb01 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 11:36:40 -0400 Subject: [PATCH 08/20] Update macros.conf Added new macros to enable complex searches for field extraction that isn't possible with standard field extractions. --- github_app_for_splunk/default/macros.conf | 31 ++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/github_app_for_splunk/default/macros.conf b/github_app_for_splunk/default/macros.conf index d8b3480..8c82925 100644 --- a/github_app_for_splunk/default/macros.conf +++ b/github_app_for_splunk/default/macros.conf @@ -4,7 +4,7 @@ definition = index=github_collectd iseval = 0 [github_source] -definition = (index="github" source="ghe_audit_log_monitoring://*") OR (index=ghes source=github_audit) +definition = (index="github" source="ghe_audit_log_monitoring://*") OR (index=ghes source=github_audit) OR (index=gitops source="github:enterprise:audit") OR (index=gh_audit_stream) iseval = 0 [github_webhooks] @@ -14,3 +14,32 @@ iseval = 0 [github_workflow_logs] definition = index="github_workflow_logs" iseval = 0 + +[devops_indexes] +definition = index="github_webhook" OR index="github_webhook2" +iseval = 0 + +[individual_commits] +definition = | spath commits{} output=commits \ +| mvexpand commits \ +| rex field=commits "(?<=\"id\"\:\")(?\w*)"\ +| rex field=commits "(?<=\"message\"\:\")(?(\w|\s)*)"\ +| rex field=commits "(?<=\"username\"\:\")(?(\w|-)*(?=\"))"\ +| rex field=commits "(?<=\"timestamp\"\:\")(?[^\"]*(?=\"))"\ +| rex field=commits "(?<=\"added\"\:\[)(?[^\]]*(?=\]))"\ +| rex field=commits "(?<=\"removed\"\:\[)(?[^\]]*(?=\]))"\ +| rex field=commits "(?<=\"modified\"\:\[)(?[^\]]*(?=\]))" +iseval = 0 + +[extract_branch_issuenumber] +definition = | eval branch = if(('ref_type'=="branch" AND 'ref'!=""), 'ref', "") \ +| eval ref = if((isnull('ref') AND isnotnull('pull_request.head.ref') AND ('eventtype'=="GitHub::PullRequest" OR 'eventtype'=="GitHub::PullRequest::Review")), 'pull_request.head.ref', if((isnull('ref') AND isnotnull('pull_request.base.ref') AND ('eventtype'=="GitHub::PullRequest" OR 'eventtype'=="GitHub::PullRequest::Review")), 'pull_request.base.ref', 'ref'))\ +| rex field="ref" "(?(?<=refs\/heads\/).*)" \ +| eval commit_branch = if((isnull('commit_branch') AND isnotnull('pull_request.head.ref') AND ('eventtype'=="GitHub::PullRequest" OR 'eventtype'=="GitHub::PullRequest::Review")), 'pull_request.head.ref', if((isnull('commit_branch') AND isnotnull('pull_request.base.ref') AND ('eventtype'=="GitHub::PullRequest" OR 'eventtype'=="GitHub::PullRequest::Review")), 'pull_request.base.ref', if((isnull('commit_branch') AND isnotnull('ref')), 'ref', 'commit_branch')))\ +| rex field="commit_branch" "(?^\d*)" +iseval = 0 + +[extract_release_push_tags] +definition = | eval ref_tags = if((isnotnull('ref') AND eventtype="GitHub::Release::Push"), ref, null())\ +| rex field="ref_tags" "(?(?<=refs\/tags\/).*)" +iseval = 0 From 171c55dbda30a3d80a2dd41a10b962370deeaa22 Mon Sep 17 00:00:00 2001 From: Doug Erkkila Date: Tue, 28 Jun 2022 11:40:30 -0400 Subject: [PATCH 09/20] Update props.conf Merge recent props changes from dev environment --- github_app_for_splunk/default/props.conf | 76 +++++++++++++++++------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/github_app_for_splunk/default/props.conf b/github_app_for_splunk/default/props.conf index 97314cc..f35a97c 100644 --- a/github_app_for_splunk/default/props.conf +++ b/github_app_for_splunk/default/props.conf @@ -1,4 +1,6 @@ [default] +FIELDALIAS-user = actor AS user + [GithubEnterpriseServerLog] DATETIME_CONFIG = LINE_BREAKER = ([\r\n]+) @@ -13,6 +15,13 @@ EXTRACT-github_log_type = \d+\:\d+\:\d+\s[\d\w\-]+\s(?.*?)\: EXTRACT-github_document_id = \"_document_id\"\:\"(?.*?)\" FIELDALIAS-source = github_log_type AS source +[GithubEnterpriseServerAuditLog] +EXTRACT-source,app,authentication_service,authentication_method,path,user,service = \<\d+\>\w+\s\d+\s\d+:\d+:\d+ (?\S+)\s+(?[^:]+)+:\s+(?\S+) : TTY=(?\S+) ; PWD=(?\S+) ; USER=(?\S+) ; COMMAND=(?.*) +EVAL-user = if(isnotnull(src_user), user, if(isnotnull(user), user, NULL)) +EVAL-signature = "Login by " + src_user + " to " + authentication_service + " service" +EVAL-action = "success" +EVAL-src = replace(source_host, "\-", ".") + [collectd_github] ADD_EXTRA_TIME_FIELDS = false ANNOTATE_PUNCT = false @@ -29,27 +38,50 @@ disabled = false pulldown_type = 1 [github_json] -DATETIME_CONFIG = CURRENT -LINE_BREAKER = ([\r\n]+) -NO_BINARY_CHECK = true -TRUNCATE = 250000 -category = Application -pulldown_type = 1 -REPORT-github_issue = extractIssueID -EXTRACT-project_card_issue_number = (.*)\"content_url\":\"(?:.*?)\/issues\/(?.*?)\"(.*) -FIELDALIAS-issueNumber = "issue.number" ASNEW issueNumber - +FIELDALIAS-dependabot = "alert.affected_package_name" AS affected_package_name "alert.external_identifier" AS cve "alert.external_reference" AS url "alert.most_recent_instance.location.path" AS alert_location_path "alert.rule.description" AS alert_description "alert.rule.security_severity_level" AS severity_level "alert.severity" AS severity eventtype AS vendor_product "repository.html_url" AS dest "repository.owner.login" AS user +EVAL-dvc = replace(host, ":\d+", "") +EVAL-signature = CASE(isnull(alert_description), UPPER(severity) + " Dependency Vulnerability on package " + affected_package_name, 1=1, alert_description) +EVAL-xref = if(isnotnull(affected_package_name), affected_package_name, alert_location_path) +FIELDALIAS-RepoAlias = "organization.login" ASNEW organization "repository.name" ASNEW repository_name +EVAL-category = if(isnotnull(alert_description), "code", if(isnotnull(affected_package_name), "dependency", "")) +disabled = false +pullrequest_base_sha = +EVAL-pullrequest_base_sha = 'pull_request.base.sha' +EVAL-pullrequest_base_user_login = 'pull_request.base.user.login' +EVAL-repository_name = 'repository.name' +KV_MODE = json +EXTRACT-commit_hash = | spath commits{} output=commits | mvexpand commits | rex field=commits "(?<=\"id\"\:\")(?\w*)" +EVAL-issue_assigned_date = if("issue.updated_at"!="" AND action="assigned", 'issue.updated_at', null()) +EVAL-issue_tags = if(isnotnull('issue.labels{}.name'), 'issue.labels{}.name', null()) +EVAL-repository_organization = if(isnotnull('organization.login'), 'organization.login', null()) +EVAL-current_priority = if('issue.labels{}.name' like "Priority%", mvfilter(match('issue.labels{}.name', "[pP]riority:\sLow|[pP]riority:\sHigh|[pP]riority:\sMedium")), null()) [github_audit] -DATETIME_CONFIG = -KV_MODE = json -LINE_BREAKER = ([\r\n]+) -NO_BINARY_CHECK = true -TIMESTAMP_FIELDS = @timestamp -TIME_FORMAT = %s%3N -TRUNCATE = 1000000 -TZ = GMT -category = Application -disabled = false -pulldown_type = 1 -FIELDALIAS-user = actor AS user +KV_MODE = JSON +FIELDALIAS-user = actor AS user "data.public_repo" AS is_public_repo org AS vendor sc4s_container AS dvc +EVAL-command = mvdedup(action) +EXTRACT-change_type = "action":"[A-z0-9_]+\.(?[^"]+)"," +EVAL-action = case(change_type="change_merge_setting", "modified", change_type="prepared_workflow_job", "modified", change_type="add_admin", "created", change_type="create", "created", change_type="invite_admin", "invite", change_type="invite_member", "invite", change_type="add_member", "modified", change_type="update_member", "modified", change_type="remove_member", "modified", change_type="grant", "modified", change_type="deauthorize", "modified", change_type="import_license_usage", "read", change_type="clone", "read", change_type="upload_license_usage", "read", change_type="repositories_added", "created", change_type="advanced_security_enabled", "modified", change_type="change_merge_setting", "modified", change_type="push", "modified", change_type="login", "logon", change_type="disabled", "modified", change_type="fetch", "read", change_type="disable", "modified", change_type="actions_enabled", "modified", change_type="add_organization", "modified", change_type="advanced_security_enabled_for_new_repos", "modified", change_type="advanced_security_policy_update", "modified", change_type="check", "read", change_type="authorized_users_teams", "modified", change_type="close", "modified", change_type="created_workflow_run", "created", change_type="enable", "modified", change_type="destroy", "deleted", change_type="enable_workflow", "modified", change_type="events_changed", "modified", change_type="completed_workflow_run", "modified", change_type="config_changed", "modified", change_type="merge", "modified", change_type="oauth_app_access_approved", "created", change_type="plan_change", "modified", change_type="remove organization", "modified", change_type="repositories_removed", "deleted", change_type="resolve", "updated", change_type="update", "updated", change_type="update_terms_of_service", "updated", change_type="remove_organization", "deleted", change_type="enable_saml", "modified", change_type="update_saml_provider_settings", "updated", change_type="disable_saml", "disabled", change_type="disable_oauth_app_restrictions", "disabled", change_type="oauth_app_access_denied", "denied", change_type="disable_two_factor_requirement", "disabled", change_type="enable_two_factor_requirement", "enable", 1=1, change_type) +EVAL-dvc = replace(host, ":\d+", "") +EXTRACT-object_path,object = "repo":"(?[^"]+)/(?