example/lava-docker: example job for using a docker shell for testing

.env

@@ -1,3 +1,3 @@
-DC_POSTGRES_IMAGE=postgres:11.2-alpine
-DC_SERVER_IMAGE=lavasoftware/lava-server:2019.04
-DC_DISPATCHER_IMAGE=lavasoftware/lava-dispatcher:2019.04
+DC_POSTGRES_IMAGE=postgres:12-alpine
+DC_SERVER_IMAGE=lavasoftware/lava-server:2020.07
+DC_DISPATCHER_IMAGE=lavasoftware/lava-dispatcher:2020.07

Makefile

@@ -1,6 +1,145 @@
+# Host address on which LAVA is accessible.
+LAVA_HOST = localhost
+# LAVA user to create/use.
+LAVA_USER = admin
+# lavacli "identity" (cached credentials name) for the above user,
+# to submit jobs, etc.
+LAVA_IDENTITY = lava-docker
+
+
+# sudo echo below is guaranteedly get a sudo password prompt and provide input
+# (may be problematic in 2nd command with "&").
 all:
+	sudo echo
+	sudo contrib/udev-forward.py -i lava-dispatcher &
 	docker-compose up
 
+stop:
+	-sudo pkill udev-forward.py
+	docker-compose stop
+
 clean:
+	-sudo pkill udev-forward.py
 	docker-compose rm -vsf
 	docker volume rm -f lava-server-pgdata lava-server-joboutput lava-server-devices lava-server-health-checks
+
+# Clean host environment to let LAVA setup run w/o issues. As an example,
+# stop ModemManager on Ubuntu, which grabs any new serial device and may
+# interfere with commnication on it.
+clean-env:
+	-sudo service apache2 stop
+	-sudo service ModemManager stop
+
+# Create various board configs for connected board(s). Supposed to be done
+# before "install" target.
+board-configs:
+	@echo "Note: you should have *all* of your boards connected to USB before running this."
+	@echo "Press Ctrl+C to break if not, or Enter to continue. Review results carefully afterwards."
+	@read dummy
+	-mv ser2net/ser2net.conf ser2net/ser2net.conf.old
+	touch ser2net/ser2net.conf
+	@echo
+	-mv contrib/LAVA.rules contrib/LAVA.rules.old
+	contrib/make-board-files.sh devices
+
+# Make any preparation steps (currently none) and build docker-compose images.
+build:
+	docker-compose build
+
+install:
+	sudo cp contrib/LAVA.rules /etc/udev/rules.d/
+	sudo cp contrib/usb-passthrough /usr/local/bin/
+	sudo udevadm control --reload
+	# This dir gets bind-mounted to dispatcher container and sub-containers
+	# it runs, to allow access to downloaded images across them.
+	sudo mkdir -p /var/lib/lava/dispatcher/tmp/
+
+lava-setup: lava-user lava-identity lava-boards
+
+lava-user:
+	@echo -n "Input password for the LAVA 'admin' user to be created: "; \
+	read passwd; \
+	test -n "$$passwd" && docker-compose exec lava-server lava-server manage users add $(LAVA_USER) --superuser --staff --passwd $$passwd || true
+	@echo
+	@echo "Now login with username: admin, passwd: (entered above) at:"
+	@echo "http://$(LAVA_HOST)/accounts/login/?next=/api/tokens/"
+	@echo "and create an auth token (long sequence of chars)."
+	@echo "(Trying to open this link in browser for you.)"
+	-xdg-open http://$(LAVA_HOST)/accounts/login/?next=/api/tokens/
+
+lava-identity:
+	@echo
+	@echo -n "Enter the auth token that you created: "; \
+	read token; \
+	test -n "$$token" && lavacli identities add --username $(LAVA_USER) --token $$token --uri http://$(LAVA_HOST)/RPC2 $(LAVA_IDENTITY) || true
+	lavacli -i $(LAVA_IDENTITY) system version
+
+lava-boards:
+	# Start with creating "virtual devices", which work in any setup,
+	# without additional hardware.
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add qemu
+	-lavacli -i $(LAVA_IDENTITY) devices add --type qemu --worker lava-dispatcher qemu-01
+	lavacli -i $(LAVA_IDENTITY) devices dict set qemu-01 devices/qemu-01.jinja2
+
+	-lavacli -i $(LAVA_IDENTITY) devices add --type qemu --worker lava-dispatcher qemu-zephyr-01
+	lavacli -i $(LAVA_IDENTITY) devices dict set qemu-zephyr-01 devices/qemu-zephyr-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add qemu-zephyr-01 qemu-zephyr
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add docker
+	-lavacli -i $(LAVA_IDENTITY) devices add --type docker --worker lava-dispatcher docker-01
+	lavacli -i $(LAVA_IDENTITY) devices dict set docker-01 devices/docker-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add docker-01 zephyr-net
+	-lavacli -i $(LAVA_IDENTITY) devices add --type docker --worker lava-dispatcher docker-02
+	lavacli -i $(LAVA_IDENTITY) devices dict set docker-02 devices/docker-generic.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add docker-02 inet
+
+	# Now create LAVA device types/device for real hardware boards. Note
+	# that these require actual device configuration files for specific
+	# boards, which are different from a developer to developer, and
+	# thus are not included in the common "lite" branch. You're expected to
+	# generate these files locally, and ideally, store in a personal fork
+	# of the repository. See the wiki page,
+	# https://collaborate.linaro.org/pages/viewpage.action?pageId=118293253#GettingStartedwithLAVA,Docker,andaFRDMK64F:-Capturinglocalboardconfigurationforsemi-automatedsetup
+	# for more info.
+	# Note that if you run directly from the "lite" branch, the setup will
+	# still succeed (because various commands below are prefixed with "-"
+	# to make them non-fatal). In this case, you will be able to use
+	# virtual devices above, but not hardware boards below.
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add musca_a
+	lavacli -i $(LAVA_IDENTITY) device-types template set musca_a device-types/musca_a.jinja2
+	# Workaround: device-type template should be on both lava-server and lava-master, but currently
+	# ends up only on lava-server after "lavacli device-types template set" above. So, we copy it
+	# to lava-master manually.
+	# Upstream issue: https://git.lavasoftware.org/lava/pkg/docker-compose/-/issues/4
+	docker cp device-types/musca_a.jinja2 lava-master:/etc/lava-server/dispatcher-config/device-types/
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add frdm-k64f
+	-lavacli -i $(LAVA_IDENTITY) devices add --type frdm-k64f --worker lava-dispatcher frdm-k64f-01
+	-lavacli -i $(LAVA_IDENTITY) devices dict set frdm-k64f-01 devices/frdm-k64f-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add frdm-k64f-01 zephyr-net
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add cc3220SF
+	-lavacli -i $(LAVA_IDENTITY) devices add --type cc3220SF --worker lava-dispatcher cc3220SF-01
+	-lavacli -i $(LAVA_IDENTITY) devices dict set cc3220SF-01 devices/cc3220SF-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add cc3220SF-01 zephyr-net
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add cc13x2-launchpad
+	-lavacli -i $(LAVA_IDENTITY) devices add --type cc13x2-launchpad --worker lava-dispatcher cc13x2-launchpad-01
+	-lavacli -i $(LAVA_IDENTITY) devices dict set cc13x2-launchpad-01 devices/cc13x2-launchpad-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add cc13x2-launchpad-01 zephyr-net
+
+	-lavacli -i $(LAVA_IDENTITY) device-types add disco-l475-iot1
+	-lavacli -i $(LAVA_IDENTITY) devices add --type disco-l475-iot1 --worker lava-dispatcher disco-l475-iot1-01
+	-lavacli -i $(LAVA_IDENTITY) devices dict set disco-l475-iot1-01 devices/disco-l475-iot1-01.jinja2
+	lavacli -i $(LAVA_IDENTITY) devices tags add disco-l475-iot1-01 zephyr-net
+
+testjob:
+	lavacli -i $(LAVA_IDENTITY) jobs submit example/micropython-interactive.job
+
+dispatcher-shell:
+	docker-compose exec lava-dispatcher bash
+
+server-shell:
+	docker-compose exec lava-server bash

README.md

@@ -1,3 +1,14 @@
+lite-lava-docker-compose
+========================
+
+This is a Linaro LITE customized fork of LAVA dockerized setup. Instructions
+for it are available in the
+[wiki](https://collaborate.linaro.org/pages/viewpage.action?pageId=118293253).
+
+Below is the original README.
+
+-------
+
 docker-compose
 ==============
 

contrib/LAVA.rules

@@ -0,0 +1,10 @@
+# Example LAVA.rules for USB passthrough to container
+#
+# Assumes that usb-passthrough script is installed in
+# /usr/local/bin/passthrough
+
+# Pass all usb devices (TTY, block, CDC) to lava-dispatcher container
+# FRDM-K64F
+ACTION=="add", ENV{ID_SERIAL_SHORT}=="MYSERIALNUM", RUN+="/usr/local/bin/usb-passthrough -a -d %E{ID_SERIAL_SHORT} -i lava-dispatcher"
+# Pass TTY through to lava-ser2net container
+ACTION=="add", SUBSYSTEM=="tty", ENV{ID_SERIAL_SHORT}=="MYSERIALNUM", RUN+="/usr/local/bin/usb-passthrough -a -d %E{ID_SERIAL_SHORT} -i lava-ser2net"

contrib/board-setup-helper.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+
+#  Copyright 2019 Linaro Limited
+#  Author: Kumar Gala <[email protected]>
+
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+
+import pyudev
+import argparse
+import os
+import pprint
+
+def ser2net(dev, ser2net_file):
+    ser2net = {}
+    port_num = 5001
+
+    with open(ser2net_file, 'r') as fd:
+        for line in fd:
+            line = line.strip()
+            values = line.split(':')
+            port = int(values.pop(0))
+            ser2net[port] = values
+
+    if len(ser2net):
+        port_num = max(ser2net.keys()) + 1
+
+    new_device = True
+
+    for k in ser2net:
+        if dev in ser2net[k][2]:
+            new_device = False
+            print("WARNING: device %s already exists in %s on port %d" % (dev, ser2net_file, k))
+            port_num = k
+
+    if new_device:
+        with open(ser2net_file, 'a') as fd:
+            fd.write("%d:telnet:0:%s:115200 8DATABITS NONE 1STOPBIT LOCAL max-connections=10\n" % (port_num, dev))
+
+    return port_num
+
+def get_device_numbers(serial_no, dev_type, board_file, ser2net_file):
+    result = None, None
+    found_tty = False
+    context = pyudev.Context()
+
+    devices = context.list_devices()
+
+    board_jinja = open(board_file, 'w')
+
+    board_jinja.write("{%% extends '%s.jinja2' %%}\n" % dev_type)
+    board_jinja.write("{%% set board_id = '%s' %%}\n" % serial_no)
+    board_jinja.write("{%% set device_info = [{'board_id': '%s'}] %%}\n" % serial_no)
+
+    for device in devices:
+        serial = device.attributes.get("serial")
+        if serial is not None and serial_no in serial.decode("utf-8"):
+            child = device.children
+            for c in child:
+                if (c.subsystem == "block"):
+                    for l in c.device_links:
+                        if "by-id" in l:
+                            board_jinja.write("{%% set usb_mass_device = '%s' %%}\n" % l)
+
+                if (c.subsystem == "tty"):
+                    for l in c.device_links:
+                        if "by-id" in l:
+                            port = ser2net(l, ser2net_file)
+                            if (port and not found_tty):
+                                board_jinja.write("{%% set connection_command = 'telnet ser2net %d' %%}\n" % port)
+                                found_tty = True #If there are multiple serial ports, connect to the first one
+            break
+
+    if dev_type == "frdm-k64f":
+        board_jinja.write("{% set resets_after_flash = false %}\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='LAVA board helper', add_help=False)
+
+    parser.add_argument("-d", "--device_serial_num", type=str, required=True,
+                        help="Devices serial number")
+
+    parser.add_argument("-t", "--device_type", type=str, required=True,
+                        help="Devices type")
+
+    parser.add_argument("-s", "--ser2net_conf", type=str, required=False,
+                        default = "./ser2net/ser2net.conf",
+                        help="ser2net configuration file")
+
+    parser.add_argument("-u", dest='udev', default=False, action='store_true')
+
+    parser.add_argument("-b", "--board_file", type=str, required=False,
+                        default = "board.jinja2",
+                        help="board jinja file")
+
+    options = parser.parse_args()
+
+    get_device_numbers(options.device_serial_num, options.device_type,
+                       options.board_file, options.ser2net_conf)
+
+    if (options.udev):
+        print('ACTION=="add", ENV{ID_SERIAL_SHORT}=="%s", RUN+="/usr/local/bin/usb-passthrough -a -d %%E{ID_SERIAL_SHORT} -i lava-dispatcher"' % options.device_serial_num)
+        print('ACTION=="add", SUBSYSTEM=="tty", ENV{ID_SERIAL_SHORT}=="%s", RUN+="/usr/local/bin/usb-passthrough -a -d %%E{ID_SERIAL_SHORT} -i lava-ser2net"' % options.device_serial_num)
+
+
+
+if __name__ == '__main__':
+    main()

contrib/make-board-files.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <devices_dir>"
+    exit 1
+fi
+
+for ser in $1/*.serial; do
+    echo "Processing: $ser"
+    devname=$(basename $ser .serial)
+    devtype=$(echo $devname | python -c "import sys; print(sys.stdin.readline().rsplit('-', 1)[0])")
+    contrib/board-setup-helper.py -d $(cat $ser) -t $devtype -b $1/$devname.jinja2 -u >>contrib/LAVA.rules
+done