From 0fe93fc3a3c9402a5d0af4415ed140df20f14953 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Fri, 28 Jul 2017 15:57:44 +0200 Subject: [PATCH] luci-app-nlbwmon: new package This commit introduces luci-app-nlbwmon, a frontend for nlbwmon, the lightweight NetLink BandWidth Montor. The nlbwmon daemon gathers per-host traffic statistics by querying netlink accounting data. Due to this approach, the executable is very small and does not rely on libpcap and CPU intensive raw sockets to monitor traffic. Depends on PR https://github.com/openwrt/packages/pull/4646 Signed-off-by: Jo-Philipp Wich --- applications/luci-app-nlbwmon/Makefile | 8 + .../luasrc/controller/nlbw.lua | 216 ++++ .../luasrc/model/cbi/nlbw/config.lua | 215 ++++ .../luasrc/view/nlbw/backup.htm | 34 + .../luasrc/view/nlbw/display.htm | 1027 +++++++++++++++++ .../root/etc/uci-defaults/40_luci-nlbwmon | 11 + 6 files changed, 1511 insertions(+) create mode 100644 applications/luci-app-nlbwmon/Makefile create mode 100644 applications/luci-app-nlbwmon/luasrc/controller/nlbw.lua create mode 100644 applications/luci-app-nlbwmon/luasrc/model/cbi/nlbw/config.lua create mode 100644 applications/luci-app-nlbwmon/luasrc/view/nlbw/backup.htm create mode 100644 applications/luci-app-nlbwmon/luasrc/view/nlbw/display.htm create mode 100644 applications/luci-app-nlbwmon/root/etc/uci-defaults/40_luci-nlbwmon diff --git a/applications/luci-app-nlbwmon/Makefile b/applications/luci-app-nlbwmon/Makefile new file mode 100644 index 0000000000..a00177f2ca --- /dev/null +++ b/applications/luci-app-nlbwmon/Makefile @@ -0,0 +1,8 @@ +include $(TOPDIR)/rules.mk + +LUCI_TITLE:=Netlink based bandwidth accounting +LUCI_DEPENDS:=+nlbwmon + +include ../../luci.mk + +# call BuildPackage - OpenWrt buildroot signature diff --git a/applications/luci-app-nlbwmon/luasrc/controller/nlbw.lua b/applications/luci-app-nlbwmon/luasrc/controller/nlbw.lua new file mode 100644 index 0000000000..4f41bb8c31 --- /dev/null +++ b/applications/luci-app-nlbwmon/luasrc/controller/nlbw.lua @@ -0,0 +1,216 @@ +-- Copyright 2017 Jo-Philipp Wich +-- Licensed to the public under the Apache License 2.0. + +module("luci.controller.nlbw", package.seeall) + +function index() + entry({"admin", "nlbw"}, firstchild(), _("Bandwidth Monitor"), 80) + entry({"admin", "nlbw", "display"}, template("nlbw/display"), _("Display"), 1) + entry({"admin", "nlbw", "config"}, cbi("nlbw/config"), _("Configuration"), 2) + entry({"admin", "nlbw", "backup"}, template("nlbw/backup"), _("Backup"), 3) + entry({"admin", "nlbw", "data"}, call("action_data"), nil, 4) + entry({"admin", "nlbw", "list"}, call("action_list"), nil, 5) + entry({"admin", "nlbw", "ptr"}, call("action_ptr"), nil, 6).leaf = true + entry({"admin", "nlbw", "download"}, call("action_download"), nil, 7) + entry({"admin", "nlbw", "restore"}, post("action_restore"), nil, 8) +end + +local function exec(cmd, args, writer) + local os = require "os" + local nixio = require "nixio" + + local fdi, fdo = nixio.pipe() + local pid = nixio.fork() + + if pid > 0 then + fdo:close() + + while true do + local buffer = fdi:read(2048) + local wpid, stat, code = nixio.waitpid(pid, "nohang") + + if writer and buffer and #buffer > 0 then + writer(buffer) + end + + if wpid and stat == "exited" then + break + end + end + elseif pid == 0 then + nixio.dup(fdo, nixio.stdout) + fdi:close() + fdo:close() + nixio.exece(cmd, args, nil) + nixio.stdout:close() + os.exit(1) + end +end + +function action_data() + local http = require "luci.http" + + local types = { + csv = "text/csv", + json = "application/json" + } + + local args = { } + local mtype = http.formvalue("type") or "json" + local delim = http.formvalue("delim") or ";" + local period = http.formvalue("period") + local group_by = http.formvalue("group_by") + local order_by = http.formvalue("order_by") + + if types[mtype] then + args[#args+1] = "-c" + args[#args+1] = mtype + else + http.status(400, "Unsupported type") + return + end + + if delim and #delim > 0 then + args[#args+1] = "-s%s" % delim + end + + if period and #period > 0 then + args[#args+1] = "-t" + args[#args+1] = period + end + + if group_by and #group_by > 0 then + args[#args+1] = "-g" + args[#args+1] = group_by + end + + if order_by and #order_by > 0 then + args[#args+1] = "-o" + args[#args+1] = order_by + end + + http.prepare_content(types[mtype]) + exec("/usr/sbin/nlbw", args, http.write) +end + +function action_list() + local http = require "luci.http" + + local fd = io.popen("/usr/sbin/nlbw -c list") + local periods = { } + + if fd then + while true do + local period = fd:read("*l") + + if not period then + break + end + + periods[#periods+1] = period + end + + fd:close() + end + + http.prepare_content("application/json") + http.write_json(periods) +end + +function action_ptr(...) + local http = require "luci.http" + local util = require "luci.util" + + http.prepare_content("application/json") + http.write_json(util.ubus("network.rrdns", "lookup", { + addrs = {...}, timeout = 3000 + })) +end + +function action_download() + local nixio = require "nixio" + local http = require "luci.http" + local sys = require "luci.sys" + local uci = require "luci.model.uci".cursor() + + local dir = uci:get_first("nlbwmon", "nlbwmon", "database_directory") + or "/var/lib/nlbwmon" + + if dir and nixio.fs.stat(dir, "type") == "dir" then + local n = "nlbwmon-backup-%s-%s.tar.gz" + %{ sys.hostname(), os.date("%Y-%m-%d") } + + http.prepare_content("application/octet-stream") + http.header("Content-Disposition", "attachment; filename=\"%s\"" % n) + exec("/bin/tar", { "-C", dir, "-c", "-z", ".", "-f", "-" }, http.write) + else + http.status(500, "Unable to find database directory") + end +end + +function action_restore() + local nixio = require "nixio" + local http = require "luci.http" + local i18n = require "luci.i18n" + local tpl = require "luci.template" + local uci = require "luci.model.uci".cursor() + + local tmp = "/tmp/nlbw-restore.tar.gz" + local dir = uci:get_first("nlbwmon", "nlbwmon", "database_directory") + or "/var/lib/nlbwmon" + + local fp + http.setfilehandler( + function(meta, chunk, eof) + if not fp and meta and meta.name == "archive" then + fp = io.open(tmp, "w") + end + if fp and chunk then + fp:write(chunk) + end + if fp and eof then + fp:close() + end + end) + + local files = { } + local tar = io.popen("/bin/tar -tzf %s" % tmp, "r") + if tar then + while true do + local file = tar:read("*l") + if not file then + break + elseif file:match("^%d%d%d%d%d%d%d%d%.db%.gz$") or + file:match("^%./%d%d%d%d%d%d%d%d%.db%.gz$") then + files[#files+1] = file + end + end + tar:close() + end + + if #files == 0 then + http.status(500, "Internal Server Error") + tpl.render("nlbw/backup", { + message = i18n.translate("Invalid or empty backup archive") + }) + return + end + + + local output = { } + + exec("/etc/init.d/nlbwmon", { "stop" }) + exec("/bin/mkdir", { "-p", dir }) + + exec("/bin/tar", { "-C", dir, "-vxzf", tmp, unpack(files) }, + function(chunk) output[#output+1] = chunk:match("%S+") end) + + exec("/bin/rm", { "-f", tmp }) + exec("/etc/init.d/nlbwmon", { "start" }) + + tpl.render("nlbw/backup", { + message = i18n.translatef( + "The following database files have been restored: %s", + table.concat(output, ", ")) + }) +end diff --git a/applications/luci-app-nlbwmon/luasrc/model/cbi/nlbw/config.lua b/applications/luci-app-nlbwmon/luasrc/model/cbi/nlbw/config.lua new file mode 100644 index 0000000000..71e096c617 --- /dev/null +++ b/applications/luci-app-nlbwmon/luasrc/model/cbi/nlbw/config.lua @@ -0,0 +1,215 @@ +-- Copyright 2017 Jo-Philipp Wich +-- Licensed to the public under the Apache License 2.0. + +local utl = require "luci.util" +local sys = require "luci.sys" +local fs = require "nixio.fs" +local ip = require "luci.ip" +local nw = require "luci.model.network" + +local s, m, period, warning, date, days, interval, ifaces, subnets, limit, prealloc, compress, generations, commit, refresh, directory, protocols + +m = Map("nlbwmon", translate("Netlink Bandwidth Monitor - Configuration"), + translate("The Netlink Bandwidth Monitor (nlbwmon) is a lightweight, efficient traffic accounting program keeping track of bandwidth usage per host and protocol.")) + +nw.init(luci.model.uci.cursor_state()) + +s = m:section(TypedSection, "nlbwmon") +s.anonymous = true +s.addremove = false +s:tab("general", translate("General Settings")) +s:tab("advanced", translate("Advanced Settings")) +s:tab("protocol", translate("Protocol Mapping"), + translate("Protocol mappings to distinguish traffic types per host, one mapping per line. The first value specifies the IP protocol, the second value the port number and the third column is the name of the mapped protocol.")) + +period = s:taboption("general", ListValue, "_period", translate("Accounting period"), + translate("Choose \"Day of month\" to restart the accounting period monthly on a specific date, e.g. every 3rd. Choose \"Fixed interval\" to restart the accounting period exactly every N days, beginning at a given date.")) + +period:value("relative", translate("Day of month")) +period:value("absolute", translate("Fixed interval")) + +period.write = function(self, cfg, val) + if period:formvalue(cfg) == "relative" then + m:set(cfg, "database_interval", interval:formvalue(cfg)) + else + m:set(cfg, "database_interval", "%s/%s" %{ + date:formvalue(cfg), + days:formvalue(cfg) + }) + end +end + +period.cfgvalue = function(self, cfg) + local val = m:get(cfg, "database_interval") or "" + if val:match("^%d%d%d%d%-%d%d%-%d%d/%d+$") then + return "absolute" + end + return "relative" +end + + +warning = s:taboption("general", DummyValue, "_warning", translate("Warning")) +warning.default = translatef("Changing the accounting interval type will invalidate existing databases!
Download backup.", luci.dispatcher.build_url("admin/nlbw/backup")) +warning.rawhtml = true + +if (m.uci:get_first("nlbwmon", "nlbwmon", "database_interval") or ""):match("^%d%d%d%d-%d%d-%d%d/%d+$") then + warning:depends("_period", "relative") +else + warning:depends("_period", "absolute") +end + + +interval = s:taboption("general", Value, "_interval", translate("Due date"), + translate("Day of month to restart the accounting period. Use negative values to count towards the end of month, e.g. \"-5\" to specify the 27th of July or the 24th of Februrary.")) + +interval.datatype = "or(range(1,31),range(-31,-1))" +interval.placeholder = "1" +interval:value("1", translate("1 - Restart every 1st of month")) +interval:value("-1", translate("-1 - Restart every last day of month")) +interval:value("-7", translate("-7 - Restart a week before end of month")) +interval.rmempty = false +interval:depends("_period", "relative") +interval.write = period.write + +interval.cfgvalue = function(self, cfg) + local val = m:get(cfg, "database_interval") + return val and tonumber(val) +end + + +date = s:taboption("general", Value, "_date", translate("Start date"), + translate("Start date of the first accounting period, e.g. begin of ISP contract.")) + +date.datatype = "dateyyyymmdd" +date.placeholder = "2016-03-15" +date.rmempty = false +date:depends("_period", "absolute") +date.write = period.write + +date.cfgvalue = function(self, cfg) + local val = m:get(cfg, "database_interval") or "" + return (val:match("^(%d%d%d%d%-%d%d%-%d%d)/%d+$")) +end + + +days = s:taboption("general", Value, "_days", translate("Interval"), + translate("Length of accounting interval in days.")) + +days.datatype = "min(1)" +days.placeholder = "30" +days.rmempty = false +days:depends("_period", "absolute") +days.write = period.write + +days.cfgvalue = function(self, cfg) + local val = m:get(cfg, "database_interval") or "" + return (val:match("^%d%d%d%d%-%d%d%-%d%d/(%d+)$")) +end + + +ifaces = s:taboption("general", Value, "_ifaces", translate("Local interfaces"), + translate("Only conntrack streams from or to any of these networks are counted.")) + +ifaces.template = "cbi/network_netlist" +ifaces.widget = "checkbox" +ifaces.nocreate = true + +ifaces.cfgvalue = function(self, cfg) + return m:get(cfg, "local_network") +end + +ifaces.write = function(self, cfg) + local item + local items = {} + for item in utl.imatch(subnets:formvalue(cfg)) do + items[#items+1] = item + end + for item in utl.imatch(ifaces:formvalue(cfg)) do + items[#items+1] = item + end + m:set(cfg, "local_network", items) +end + + +subnets = s:taboption("general", DynamicList, "_subnets", translate("Local subnets"), + translate("Only conntrack streams from or to any of these subnets are counted.")) + +subnets.datatype = "ipaddr" + +subnets.cfgvalue = function(self, cfg) + local subnet + local subnets = {} + for subnet in utl.imatch(m:get(cfg, "local_network")) do + subnet = ip.new(subnet) + subnets[#subnets+1] = subnet and subnet:string() + end + return subnets +end + +subnets.write = ifaces.write + + +limit = s:taboption("advanced", Value, "database_limit", translate("Maximum entries"), + translate("The maximum amount of entries that should be put into the database, setting the limit to 0 will allow databases to grow indefinitely.")) + +limit.datatype = "uinteger" +limit.placeholder = "10000" + +prealloc = s:taboption("advanced", Flag, "database_prealloc", translate("Preallocate database"), + translate("Whether to preallocate the maximum possible database size in memory. This is mainly useful for memory constrained systems which might not be able to satisfy memory allocation after longer uptime periods.")) + +prealloc:depends({["database_limit"] = "0", ["!reverse"] = true }) + + +compress = s:taboption("advanced", Flag, "database_compress", translate("Compress database"), + translate("Whether to gzip compress archive databases. Compressing the database files makes accessing old data slightly slower but helps to reduce storage requirements.")) + +compress.default = compress.enabled + + +generations = s:taboption("advanced", Value, "database_generations", translate("Stored periods"), + translate("Maximum number of accounting periods to keep, use zero to keep databases forever.")) + +generations.datatype = "uinteger" +generations.placeholder = "10" + + +commit = s:taboption("advanced", Value, "commit_interval", translate("Commit interval"), + translate("Interval at which the temporary in-memory database is committed to the persistent database directory.")) + +commit.placeholder = "24h" +commit:value("24h", translate("24h - least flash wear at the expense of data loss risk")) +commit:value("12h", translate("12h - compromise between risk of data loss and flash wear")) +commit:value("10m", translate("10m - frequent commits at the expense of flash wear")) +commit:value("60s", translate("60s - commit minutely, useful for non-flash storage")) + + +refresh = s:taboption("advanced", Value, "refresh_interval", translate("Refresh interval"), + translate("Interval at which traffic counters of still established connections are refreshed from netlink information.")) + +refresh.placeholder = "30s" +refresh:value("30s", translate("30s - refresh twice per minute for reasonably current stats")) +refresh:value("5m", translate("5m - rarely refresh to avoid frequently clearing conntrack counters")) + + +directory = s:taboption("advanced", Value, "database_directory", translate("Database directory"), + translate("Database storage directory. One file per accounting period will be placed into this directory.")) + +directory.placeholder = "/var/lib/nlbwmon" + + +protocols = s:taboption("protocol", TextValue, "_protocols") +protocols.rows = 50 + +protocols.cfgvalue = function(self, cfg) + return fs.readfile("/usr/share/nlbwmon/protocols") +end + +protocols.write = function(self, cfg, value) + fs.writefile("/usr/share/nlbwmon/protocols", (value or ""):gsub("\r\n", "\n")) +end + +protocols.remove = protocols.write + + +return m diff --git a/applications/luci-app-nlbwmon/luasrc/view/nlbw/backup.htm b/applications/luci-app-nlbwmon/luasrc/view/nlbw/backup.htm new file mode 100644 index 0000000000..ea2e0f05cf --- /dev/null +++ b/applications/luci-app-nlbwmon/luasrc/view/nlbw/backup.htm @@ -0,0 +1,34 @@ +<%# + Copyright 2017 Jo-Philipp Wich + Licensed to the public under the Apache License 2.0. +-%> + +<%+header%> + + + +

<%:Netlink Bandwidth Monitor - Backup / Restore %>

+ +
+ <%:Restore Database Backup%> +

+

" enctype="multipart/form-data"> + + + +
+ + <% if message then %> +
<%=message%>
+ <% end %> +

+ + <%:Download Database Backup%> +

+

"> + +
+

+
+ +<%+footer%> diff --git a/applications/luci-app-nlbwmon/luasrc/view/nlbw/display.htm b/applications/luci-app-nlbwmon/luasrc/view/nlbw/display.htm new file mode 100644 index 0000000000..7db47852e6 --- /dev/null +++ b/applications/luci-app-nlbwmon/luasrc/view/nlbw/display.htm @@ -0,0 +1,1027 @@ +<%# + Copyright 2017 Jo-Philipp Wich + Licensed to the public under the Apache License 2.0. +-%> + +<% css = [[ + + #chartjs-tooltip { + opacity: 0; + position: absolute; + background: rgba(0, 0, 0, .7); + color: white; + padding: 3px; + border-radius: 3px; + -webkit-transition: all .1s ease; + transition: all .1s ease; + pointer-events: none; + -webkit-transform: translate(-50%, 0); + transform: translate(-50%, 0); + z-index: 200; + } + + #chartjs-tooltip.above { + -webkit-transform: translate(-50%, -100%); + transform: translate(-50%, -100%); + } + + #chartjs-tooltip.above:before { + border: solid; + border-color: #111 transparent; + border-color: rgba(0, 0, 0, .8) transparent; + border-width: 8px 8px 0 8px; + bottom: 1em; + content: ""; + display: block; + left: 50%; + top: 100%; + position: absolute; + z-index: 99; + -webkit-transform: translate(-50%, 0); + transform: translate(-50%, 0); + } + + table { + border: 1px solid #999; + border-collapse: collapse; + margin: 0 0 2px !important; + } + + th, td, table table td { + border: 1px solid #999; + text-align: right; + padding: 1px 3px !important; + white-space: nowrap; + } + + tbody td { + border-bottom-color: #ccc; + } + + tbody td[rowspan] { + border-bottom-color: #999; + } + + tbody tr:last-child td { + border-bottom-color: #999; + } + + + .pie { + width: 200px; + display: inline-block; + margin: 20px; + } + + .pie label { + font-weight: bold; + font-size: 14px; + display: block; + margin-bottom: 10px; + text-align: center; + } + + .kpi { + display: inline-block; + margin: 80px 20px 20px; + vertical-align: top; + } + + .kpi ul { + list-style: none; + } + + .kpi li { + margin: 10px; + display: none; + } + + .kpi big { + font-weight: bold; + } + + #detail-bubble { + position: absolute; + opacity: 0; + visibility: hidden; + } + + #detail-bubble.in { + opacity: 1; + visibility: visible; + transition: opacity 0.5s; + } + + #detail-bubble > div { + border: 1px solid #ccc; + border-radius: 2px; + padding: 5px; + background: #fcfcfc; + } + + #detail-bubble .head { + text-align: center; + white-space: nowrap; + position: relative; + } + + #detail-bubble .head .dismiss { + top: 0; + right: 0; + width: 20px; + line-height: 20px; + text-align: center; + text-decoration: none; + font-weight: bold; + color: #000; + position: absolute; + font-size: 20px; + } + + #detail-bubble .pie { + width: 100px; + margin: 5px; + } + + #detail-bubble .kpi { + margin: 40px 5px 5px; + font-size: smaller; + text-align: left; + } + + #detail-bubble .kpi ul { + margin: 0; + } + + #bubble-arrow { + border: 1px solid #ccc; + border-width: 1px 0 0 1px; + background: #fcfcfc; + width: 15px; + height: 15px; + position: absolute; + left: 0; + top: -8px; + transform: rotate(45deg); + margin: 0 0 0 -8px; + } + + tr.active > td { + border-bottom: 2px solid red; + } + + tr.active > td.active { + border: 2px solid red; + border-bottom: none; + } + + td.detail { + border: 2px solid red; + border-top: none; + opacity: 0; + transition: opacity 0.5s; + } + + td.detail.in { + opacity: 1; + } + + th.hostname, + td.hostname { + text-align: left; + } + +]] -%> + +<%+header%> + + + + + +

<%:Netlink Bandwidth Monitor%>

+ +
+ +
+
+ × +
+ + +
+
+ + +
+
+
    +
  • <%_Hostname: example.org%>
    • +
  • <%_Vendor: Example Corp.%>
    • +
+
+
+
+
+
+ +
+ +

+ <%:Select accounting period:%> + +

+ +
+ + + +
+
+
+ + +
+ +
+ + +
+ +
+
    +
  • <%_0 hosts%>
    • +
  • <%_0 download%>
    • +
  • <%_0 upload%>
    • +
  • <%_0 connections%>
    • +
+
+
+ + + + + + + + +
<%:Host%><%:MAC%><%:Connections%><%:Download (Bytes / Packets)%><%:Upload (Bytes / Packets)%>
+
+ + + + + + + + + +<%+footer%> diff --git a/applications/luci-app-nlbwmon/root/etc/uci-defaults/40_luci-nlbwmon b/applications/luci-app-nlbwmon/root/etc/uci-defaults/40_luci-nlbwmon new file mode 100644 index 0000000000..c9771779ee --- /dev/null +++ b/applications/luci-app-nlbwmon/root/etc/uci-defaults/40_luci-nlbwmon @@ -0,0 +1,11 @@ +#!/bin/sh + +uci -q batch <<-EOF >/dev/null + delete ucitrack.@nlbwmon[-1] + add ucitrack nlbwmon + set ucitrack.@nlbwmon[-1].init=nlbwmon + commit ucitrack +EOF + +rm -f /tmp/luci-indexcache +exit 0 -- 2.30.2