diff --git a/deploy/k8s-microk8s/sc4s-ds.yaml b/deploy/k8s-microk8s/sc4s-ds.yaml new file mode 100644 index 0000000..dd8c05c --- /dev/null +++ b/deploy/k8s-microk8s/sc4s-ds.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: splunk-sc4s + labels: + app: sc4s +spec: + selector: + matchLabels: + name: splunk-sc4s + template: + metadata: + labels: + name: splunk-sc4s + spec: + tolerations: + # this toleration is to have the daemonset runnable on master nodes + # remove it if your masters can't run pods + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: sc4s + image: localhost:32000/scs:latest + ports: + - containerPort: 514 + envFrom: + - configMapRef: + name: sc4s-env-file + env: + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-s1-standalone-secrets + key: hec_token + - name: SC4S_SNMP_TRAP_COLLECT + value: "no" + - name: SC4S_CONTAINER_HOST + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SC4S_RUNTIME_ENV + value: "k8s" + livenessProbe: + httpGet: + path: /healthz + port: 8080 + # initialDelaySeconds: 15 + periodSeconds: 3 + startupProbe: + httpGet: + path: /healthz + port: 8080 + failureThreshold: 30 + periodSeconds: 10 + volumeMounts: + - name: syslog-var + mountPath: "/opt/syslog-ng/var" + - name: sc4s-context + mountPath: /opt/syslog-ng/etc/conf.d/configmap/context + terminationGracePeriodSeconds: 600 + volumes: + - name: syslog-var + persistentVolumeClaim: + claimName: splunk-sc4s-pvc + - name: sc4s-context + configMap: + name: sc4s-context-config diff --git a/deploy/k8s-microk8s/sc4s-infra.yaml b/deploy/k8s-microk8s/sc4s-infra.yaml new file mode 100644 index 0000000..a06203c --- /dev/null +++ b/deploy/k8s-microk8s/sc4s-infra.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sc4s-env-file +data: + SPLUNK_HEC_URL: https://splunk-s1-standalone-headless:8088 + SC4S_DEST_SPLUNK_HEC_TLS_VERIFY: "yes" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sc4s-context-config +data: + # example of a simple property defined using --from-literal + compliance_meta_by_source.conf: |- + filter f_test_test { + # host("something-*" type(glob)) or + # netmask(169.254.100.0/24) + host("cannot_ever_happen") + }; + compliance_meta_by_source.csv: |- + f_test_test,.splunk.index,"will_never_happen_index" + f_test_test,fields.compliance,"pci" + host.csv: |- + 169.254.0.2,HOST,foo.example + splunk_metadata.csv: |- + bluecoat_proxy,index,netproxy + brocade_syslog,index,netops + ArcSight_ArcSight,index,main + Cyber-Ark_Vault,index,netauth + CyberArk_PTA,index,main + Incapsula_SIEMintegration,index,netwaf + Microsoft_Microsoft Windows,index,oswinsec + Microsoft_System or Application Event,index,oswin + checkpoint_splunk,index,netops + checkpoint_splunk_dlp,index,netdlp + checkpoint_splunk_email,index,email + checkpoint_splunk_firewall,index,netfw + checkpoint_splunk_ids,index,netids + checkpoint_splunk_os,index,netops + checkpoint_splunk_sessions,index,netops + checkpoint_splunk_web,index,netproxy + checkpoint_splunk,index,netops + checkpoint_splunk,index,netops + cisco_apic_acl,index,netfw + cisco_apic_events,index,netops + cisco_acs,index,netauth + cisco_asa,index,netfw + cisco_ftd,index,netfw + cisco_ios,index,netops + cisco_ise,index,netauth + cisco_meraki,index,netfw + cisco_nx_os,index,netops + cisco_ucm,index,main + cisco_wsa,index,netproxy + dell_rsa_secureid,index,netauth + citrix_netscaler,index,netfw + local_example,index,main + forcepoint_webprotect,index,netproxy + f5_bigip,index,netops + f5_bigip_access_json,index,netops + f5_bigip_irule,index,netops + f5_bigip_asm,index,netwaf + f5_bigip_nix,index,netops + fortinet_fortios_event,index,netops + fortinet_fortios_log,index,netops + fortinet_fortios_traffic,index,netfw + fortinet_fortios_utm,index,netids + fortinet_fortiweb_attack,index,netids + fortinet_fortiweb_event,index,netops + fortinet_fortiweb_log,index,netops + fortinet_fortiweb_traffic,index,netfw + infoblox_dns,index,netdns + infoblox_dhcp,index,netipam + infoblox_threat,index,netids + juniper_idp,index,netids + juniper_structured,index,netops + juniper_idp_structured,index,netids + juniper_junos_fw_structured,index,netfw + juniper_junos_ids_structured,index,netids + juniper_junos_utm_structured,index,netfw + juniper_junos_aamw_structured,index,netfw + juniper_junos_secintel_structured,index,netfw + juniper_junos_fw,index,netfw + juniper_junos_ids,index,netids + juniper_junos_utm,index,netfw + juniper_netscreen,index,netfw + juniper_legacy,index,netops + mcafee_epo,index,epav + nix_syslog,index,osnix + pan_traffic,index,netfw + pan_threat,index,netproxy + pan_system,index,netops + pan_config,index,netops + pan_hipmatch,index,main + pan_correlation,index,main + pan_userid,index,netauth + pan_unknown,index,netops + pfsense,index,netops + pfsense_filterlog,index,netfw + proofpoint_pps_filter,index,email + proofpoint_pps_sendmail,index,email + sc4s_events,index,main + sc4s_fallback,index,main + sc4s_metrics,index,em_metrics + symantec_ep,index,epav + symantec_brightmail,index,email + ubiquiti_unifi,index,netops + ubiquiti_unifi_fw,index,netfw + ubiquiti_unifi_link,index,netops + ubiquiti_unifi_sudo,index,netops + ubiquiti_unifi_switch,index,netops + ubiquiti_unifi_threat,index,netids + ubiquiti_unifi_wireless,index,netops + vmware_esx,index,main + vmware_horizon,index,main + vmware_nsx,index,main + vmware_vcenter,index,main + zscaler_alerts,index,netops + zscaler_dns,index,netdns + zscaler_fw,index,netfw + zscaler_web,index,netproxy + zscaler_zia_audit,index,netops + zscaler_zia_sandbox,index,main + zscaler_lss,index,netproxy + vendor_product_by_source.conf: |- + filter f_test_test { + host("testvp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_null_queue { + netmask(169.254.100.0/24) + }; + filter f_brocade_syslog { + host("test_brocade-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_citrix_netscaler { + host("test_ctitrixns-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_dell_rsa_secureid { + host("test_rsasecureid*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_juniper_netscreen { + host("jnpns-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_cisco_meraki { + host("testcm-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_cisco_wsa{ + host("cisco_wsa" type(glob)) + }; + filter f_cisco_wsa11_7{ + host("cisco_wsa11_7" type(glob)) + }; + filter f_cisco_nx_os { + host("csconx-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_f5_bigip { + host("test_f5-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_infoblox { + host("vib-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_pfsense { + host("pfsense-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_schneider_apc { + host("test_apc-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_ubiquiti_unifi_fw { + host("usg-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_tzfixhst { + host("tzfhst-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + filter f_tzfixny { + host("tzfny-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) + }; + vendor_product_by_source.csv: |- + f_test_test,sc4s_vendor_product,"test_test" + f_brocade_syslog,sc4s_vendor_product,"brocade_syslog" + f_null_queue,sc4s_vendor_product,"null_queue" + f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" + f_cisco_wsa,sc4s_vendor_product,"cisco_wsa" + f_cisco_wsa11_7,sc4s_vendor_product,"cisco_wsa11_7" + f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler" + f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid" + f_f5_bigip,sc4s_vendor_product,"f5_bigip" + f_infoblox,sc4s_vendor_product,"infoblox" + f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" + f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" + f_pfsense,sc4s_vendor_product,"pfsense" + f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" + f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" + f_schneider_apc,sc4s_vendor_product,"schneider_apc" + f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" + f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu" + f_tzfixny,sc4s_time_zone,"America/New_York" + +--- + +--- +apiVersion: v1 +kind: Service +metadata: + name: sc4s-ext-tcp + annotations: + metallb.universe.tf/allow-shared-ip: sc4s +spec: + ports: + - port: 514 + targetPort: 514 + protocol: TCP + selector: + app: sc4s + type: LoadBalancer + externalTrafficPolicy: Local +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: splunk-sc4s-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500M +--- +apiVersion: v1 +kind: Service +metadata: + name: sc4s-ext-udp + annotations: + metallb.universe.tf/allow-shared-ip: sc4s +spec: + ports: + - port: 514 + targetPort: 514 + protocol: UDP + selector: + app: sc4s + type: LoadBalancer + externalTrafficPolicy: Local +--- + diff --git a/docs/gettingstarted/k8s-microk8s.md b/docs/gettingstarted/k8s-microk8s.md new file mode 100644 index 0000000..2c8091d --- /dev/null +++ b/docs/gettingstarted/k8s-microk8s.md @@ -0,0 +1,77 @@ + +# Install MicroK8s - ALPHA + +SUPPORT NOTICE DEPLOYMENT VIA K8S is ALPHA is and not officially supported for production + +The SC4S deployment model with Microk8s uses specific features of this distribution of k8s. +While this may be reproducable with other distributions such an undertaking requires more advanced +awareness and responsibility for the administrator. + +* (metalLB) ensure source IP is preserved +* Bring any operating system (window/centos/rhel/ubuntu/debian) + +This configuration requires as least 2 IP addressed one for host and one for the internal load balancer. +We suggest allocation of 3 ip addresses for the host and 5-10 addresses for later use + +# FAQ + +Question: Why is this "load balancer" ok but others are not? +Answer: While we are using a load balancer with one instance per host the traffic is restricted +to the entry node and one instance of sc4s will run per node. This limits the function of MetalLB to +the same function as a Cluster Manager. + +```bash +#we need to have a normal install of kubectl because of operator scripts +sudo snap install kubectl --classic +# Basic setup of k8s +sudo snap install microk8s --classic --channel=1.18/stable +sudo usermod -a -G microk8s $USER +sudo chown -f -R $USER ~/.kube + +su - $USER +microk8s status --wait-ready +microk8s enable dns metallb rbac storage +microk8s status --wait-ready +mkdir ~/.kube +#tell the default install of kubectl how to talk to our cluster +microk8s.config > $HOME/.kube/config +# +``` + +# Install SC4S + +```bash +git clone https://github.com/splunk/splunk-connect-for-syslog.git +cd splunk-connect-for-syslog +kubectl create ns sc4s +kubectl apply -n sc4s -f deploy/k8s-microk8s/sc4s-infra.yaml +# Important modify the following command to use the correct token +echo -n 'A8AE530F-73C6-E990-704A-963E3623F4D0' > hec_token.txt +kubectl create -n sc4s secret generic sc4s-secrets --from-file=hec_token=./hec_token.txt +rm hec_token.txt +# Edit the values for SPLUNK_HEC_URL and SC4S_DEST_SPLUNK_HEC_TLS_VERIFY +kubectl edit -n sc4s configmap sc4s-env-file +# Deploy sc4s +kubectl apply -n sc4s -f deploy/k8s-microk8s/sc4s-deploy.yaml +# Watch pods use ctrl + c to terminate when running +kubectl get -n sc4s pods -w +# Optional get logs replace with pod name above +kubectl -n sc4s logs splunk-sc4s-22rr6 +``` + +Check Splunk for events + +# Change configuration + +Note change change to the following config will trigger a restart of the container + +```bash +kubectl edit configmap sc4s-env-file +kubectl edit configmap sc4s-context-config +``` + +# Setup for HA with multiple nodes + +See https://microk8s.io/docs/high-availability + +Note three identically size nodes are required for HA \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index df3406a..0d7ff19 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,7 @@ nav: - "Docker CE + systemd": "gettingstarted/docker-systemd-general.md" - "Docker CE + Swarm": "gettingstarted/docker-swarm-general.md" - "Docker CE + Swarm RHEL 7.7": "gettingstarted/docker-swarm-rhel7.md" + - "MicroK8s + Linux": "gettingstarted/k8s-microk8s.md" - "Bring your own Envionment": "gettingstarted/byoe-rhel7.md" - "Quickstart Guide": "gettingstarted/quickstart_guide.md" - Configuration: "configuration.md" @@ -15,7 +16,7 @@ nav: - Development: "developing/index.md" - Sources: - About: sources/index.md - - Brocade: sources/Brocade/index.md + - Brocade: sources/Brocade/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md - Citrix: sources/Citrix/index.md diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index b9382b2..02250b8 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -48,26 +48,39 @@ trap 'kill ${!}; term_handler' SIGTERM mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ + + cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context for file in /opt/syslog-ng/etc/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done +if [ "$SC4S_RUNTIME_ENV" == "k8s" ] +then + mkdir -p /opt/syslog-ng/etc/conf.d/configmap/context/ + mkdir -p /opt/syslog-ng/etc/conf.d/configmap/config/ + # Add new entries + temp_file=$(mktemp) + awk '{print $0}' /opt/syslog-ng/etc/conf.d/configmap/context/splunk_metadata.csv /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u > $temp_file + cp -f $temp_file /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv + +else + # splunk_index.csv updates + # Remove comment headers from existing config + touch /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv + if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then + LEGACY_SPLUNK_INDEX_FILE=/opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv + fi -# splunk_index.csv updates -# Remove comment headers from existing config -touch /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv -if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then - LEGACY_SPLUNK_INDEX_FILE=/opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv -fi -# Add new entries -temp_file=$(mktemp) -awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u > $temp_file -cp -f $temp_file /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv -# We don't need this file any longer -rm -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv.example || true -if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then - cp -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.deprecated - rm /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv -fi -cp --verbose -R -f /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ + # Add new entries + temp_file=$(mktemp) + awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u > $temp_file + cp -f $temp_file /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv + # We don't need this file any longer + rm -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv.example || true + if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then + cp -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.deprecated + rm /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv + fi + cp --verbose -R -f /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +fi mkdir -p /opt/syslog-ng/var/log # Test HEC Connectivity