feat: added lighty-yang-validator, added validation benchmarking

Dockerfile

@@ -5,18 +5,23 @@ ARG YANGCATALOG_CONFIG_PATH
 ARG CONFD_VERSION
 ARG YANGLINT_VERSION
 ARG XYM_VERSION
+ARG VALIDATOR_JDK_VERSION
+ARG MAVEN_VERSION
+ARG LYV_VERSION
 
 ENV YANG_ID "$YANG_ID"
 ENV YANG_GID "$YANG_GID"
 ENV YANGCATALOG_CONFIG_PATH "$YANGCATALOG_CONFIG_PATH"
 ENV CONFD_VERSION "$CONFD_VERSION"
 ENV YANGLINT_VERSION "$YANGLINT_VERSION"
 ENV XYM_VERSION "$XYM_VERSION"
-
+ENV VALIDATOR_JDK_VERSION "$VALIDATOR_JDK_VERSION"
+ENV MAVEN_VERSION "$MAVEN_VERSION"
+ENV LYV_VERSION "$LYV_VERSION"
 ENV VIRTUAL_ENV=/home/yangvalidator/yang-extractor-validator
 
 RUN apt-get -y update
-RUN apt-get install -y clang cmake git gnupg2 gunicorn openssh-client rsyslog
+RUN apt-get install -y clang cmake git gnupg2 gunicorn openssh-client rsyslog unzip wget
 
 # Create 'yang' user and group
 RUN groupadd -g ${YANG_GID} -r yang && useradd --no-log-init -r -g yang -u ${YANG_ID} -d /home yang
@@ -27,9 +32,33 @@ RUN mkdir -p /home/libyang/build
 WORKDIR /home/libyang/build
 RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. && make && make install
 
+WORKDIR /home
+# Set up java and maven
+RUN wget https://download.oracle.com/java/${VALIDATOR_JDK_VERSION}/latest/jdk-${VALIDATOR_JDK_VERSION}_linux-x64_bin.deb
+RUN dpkg -i jdk-${VALIDATOR_JDK_VERSION}_linux-x64_bin.deb
+
+RUN wget https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip
+RUN unzip apache-maven-${MAVEN_VERSION}-bin.zip -d /usr/local/bin/
+RUN mv /usr/local/bin/apache-maven-${MAVEN_VERSION} /usr/local/bin/maven
+
+# Set up Maven settings for building lighty-yang-validator
+RUN mkdir -p /root/.m2
+RUN wget -O /root/.m2/settings.xml https://raw.githubusercontent.com/opendaylight/odlparent/master/settings.xml
+
+# Set up lighty-yang-validator
+RUN mkdir -p /home/lyv/src
+WORKDIR /home/lyv/src
+RUN git clone -b ${LYV_VERSION} --depth 1 https://github.com/PANTHEONtech/lighty-yang-validator.git
+WORKDIR /home/lyv/src/lighty-yang-validator
+RUN /usr/local/bin/maven/bin/mvn clean install
+RUN unzip ./target/lighty-yang-validator-*-bin.zip -d /home/lyv/
+RUN mv /home/lyv/lighty-yang-validator-*/* /home/lyv/
+RUN chmod +x /home/lyv/lyv
+
 RUN sed -i "/imklog/s/^/#/" /etc/rsyslog.conf
 RUN rm -rf /var/lib/apt/lists/*
 
+WORKDIR /home
 RUN pip3 install --upgrade pip
 COPY ./yang-validator-extractor/requirements.txt .
 RUN pip3 install -r requirements.txt

yangvalidator/v2/confdParser.py

@@ -19,6 +19,7 @@
 
 import logging
 import os
+from time import perf_counter
 import typing as t
 from datetime import datetime, timezone
 from subprocess import CalledProcessError, call, check_output
@@ -53,8 +54,10 @@ def parse_module(self):
         outfp = open(self.__confdc_outfile, 'w+')
         cresfp = open(self.__confdc_resfile, 'w+')
 
-        self.LOG.info(f'Starting to confd parse use command {" ".join(self.__confdc_command)}')
+        self.LOG.info(f'Starting confd parse using command {" ".join(self.__confdc_command)}')
+        t0 = perf_counter()
         status = call(self.__confdc_command, stdout=outfp, stderr=cresfp)
+        td = perf_counter()-t0
 
         confdc_output = confdc_stderr = ''
         if os.path.isfile(self.__confdc_outfile):
@@ -76,5 +79,6 @@ def parse_module(self):
         confdc_res['version'] = self.VERSION
         confdc_res['code'] = status
         confdc_res['command'] = ' '.join(self.__confdc_command)
+        confdc_res['validation_time'] = td
 
         return confdc_res

yangvalidator/v2/lyvParser.py

@@ -0,0 +1,83 @@
+# Copyright The IETF Trust 2021, All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'Miroslav Kovac'
+__copyright__ = 'Copyright The IETF Trust 2021, All Rights Reserved'
+__license__ = 'Apache License, Version 2.0'
+__email__ = '[email protected]'
+
+import logging
+import os
+from pathlib import Path
+import typing as t
+from datetime import datetime, timezone
+from subprocess import CalledProcessError, run, check_output
+from time import perf_counter
+
+
+class LyvParser:
+    """
+    Cover the parsing of the module with lighty-yang-validator parser and validator
+    """
+
+    LYV_CMD = '/home/lyv/lyv'
+    try:
+        VERSION = (
+            check_output(f'{LYV_CMD} -v', shell=True).decode('utf-8').replace("Version: ", "").split("\n")[0].rstrip()
+        )
+    except CalledProcessError:
+        VERSION = 'undefined'
+    LOG = logging.getLogger(__name__)
+
+    def __init__(self, context_directories, file_name: str, working_directory: str):
+        self._working_directory = working_directory
+
+        # Build the command
+        cmds = [self.LYV_CMD, os.path.join(working_directory, file_name)]
+        if context_directories:
+            cmds.extend(['-p', ":".join(context_directories)])
+        self._lyv_cmd = cmds
+
+    def parse_module(self):
+        lyv_res: t.Dict[str, t.Union[str, int]] = {'time': datetime.now(timezone.utc).isoformat()}
+
+        self.LOG.info(f'Starting lyv parse using command {" ".join(self._lyv_cmd)}')
+        t0 = perf_counter()
+        result = run(self._lyv_cmd, capture_output=True, text=True)
+        td = perf_counter()-t0
+
+        # LYV returns everything in stdout, even errors. The only thing other than an error it can
+        # return is a single line with the html output name, which we don't care about (for now).
+        _HTML_GEN_STRING = "html generated to "
+        stdout = result.stdout
+        for line in stdout.splitlines():
+            if line.startswith(_HTML_GEN_STRING):
+                htmlpath = Path(line.replace(_HTML_GEN_STRING, ""))
+                if htmlpath.is_file():
+                    # Remove unnecessary HTML report, but maybe we could display it somehow instead?
+                    htmlpath.unlink()
+                stdout = stdout.replace(line, "", 1)
+                break
+
+        # Post-process results
+        dirname = os.path.dirname(self._working_directory)
+        lyv_res['stdout'] = stdout.replace(f'{dirname}/', '').strip()
+        lyv_res['stderr'] = result.stderr.replace(f'{dirname}/', '').strip()
+        lyv_res['name'] = 'lighty-yang-validator'
+        lyv_res['version'] = self.VERSION
+        lyv_res['code'] = result.returncode
+        lyv_res['command'] = ' '.join(self._lyv_cmd)
+        lyv_res['validation_time'] = td
+
+        return lyv_res

yangvalidator/v2/pyangParser.py

@@ -21,6 +21,7 @@
 import io
 import logging
 import os
+from time import perf_counter
 import typing as t
 from datetime import datetime, timezone
 
@@ -77,7 +78,10 @@ def parse_module(self):
                 self.LOG.info('no module provided')
             self.__ctx.add_module(self.__infile, module)
 
+        t0 = perf_counter()
         self.__ctx.validate()
+        td = perf_counter()-t0
+
         pyang_stderr, pyang_output = self.__print_pyang_output()
         dirname = os.path.dirname(self.__working_directory)
 
@@ -87,6 +91,7 @@ def parse_module(self):
         pyang_res['version'] = self.VERSION
         pyang_res['code'] = 0 if not pyang_stderr else 1
         pyang_res['command'] = ' '.join(self.__pyang_command)
+        pyang_res['validation_time'] = td
         restore_statements()
         del self.__ctx
         return pyang_res

yangvalidator/v2/views.py

@@ -26,6 +26,8 @@
 import random
 import shutil
 import string
+
+from time import perf_counter
 import typing as t
 from zipfile import ZipFile
 
@@ -41,6 +43,7 @@
 from yangvalidator.v2.xymParser import XymParser
 from yangvalidator.v2.yangdumpProParser import YangdumpProParser
 from yangvalidator.v2.yanglintParser import YanglintParser
+from yangvalidator.v2.lyvParser import LyvParser
 
 logger = logging.getLogger(__name__)
 
@@ -70,6 +73,7 @@ def validate(request: WSGIRequest, xym_result=None, json_body=None):
     Validate yang module using 4 different validators. Yanglint, Pyang, Confdc, Yumadump-pro.
     Check if they are valid modules according to these validators and if not return problems that
     occurred while validating by each parser.
+
     :param json_body: json body sent from other function
     :param request: request sent from user
     :return: HTTP response with validated yang modules
@@ -159,8 +163,13 @@ def validate(request: WSGIRequest, xym_result=None, json_body=None):
                 (ConfdParser, 'confd'),
                 (YanglintParser, 'yanglint'),
                 (YangdumpProParser, 'yangdump-pro'),
+                (LyvParser, 'lighty-yang-validator'),
             ):
+                t0 = perf_counter()
                 parser_results = parser([working_dir], module_to_validate, working_dir).parse_module()
+                t1 = perf_counter()
+                parser_results['total_time'] = round((t1-t0)*1000, 2)
+                parser_results['validation_time'] = round(parser_results['validation_time']*1000, 2)
                 results[module_to_validate][name] = parser_results
     except Exception as e:
         results['error'] = f'Failed to parse a document - {e}'
@@ -449,6 +458,7 @@ def versions(request: WSGIRequest):
             'xym-version': XymParser.VERSION,
             'yangdump-version': YangdumpProParser.VERSION,
             'yanglint-version': YanglintParser.VERSION,
+            'lighty-yang-validator-version': LyvParser.VERSION,
         },
     )
 
@@ -465,6 +475,7 @@ def swagger(request: WSGIRequest):
 def try_validate_and_load_data(request: WSGIRequest):
     """
     Check if request is POST and try to parse byte string to json format
+
     :param request: request sent from user
     :return: Parsed json string
     """
@@ -476,6 +487,7 @@ def try_validate_and_load_data(request: WSGIRequest):
 def create_random_suffix() -> str:
     """
     Create random suffix to create new temp directory
+
     :return: suffix of random 8 letters
     """
     letters = string.ascii_letters

yangvalidator/v2/xymParser.py

@@ -21,6 +21,7 @@
 import sys
 from datetime import datetime, timezone
 from io import StringIO
+from time import perf_counter
 
 from xym import __version__ as xym_version
 from xym import xym
@@ -44,6 +45,7 @@ def __init__(self, source, working_directory):
         self.__source = source
 
     def parse_and_extract(self):
+        t0 = perf_counter()
         extracted_models = xym.xym(
             source_id=self.__source,
             dstdir=self.__working_directory,
@@ -53,6 +55,8 @@ def parse_and_extract(self):
             debug_level=0,
             force_revision_regexp=True,
         )
+        td = perf_counter()-t0
+
         xym_res = {'time': datetime.now(timezone.utc).isoformat()}
         sys.stderr = self.__stderr_
         sys.stdout = self.__stdout_
@@ -65,4 +69,5 @@ def parse_and_extract(self):
             f'xym.xym(source_id="{self.__source}", dstdir="{self.__working_directory}", srcdir="", '
             'strict=True, strict_examples=False, debug_level=0, force_revision_regexp=True)'
         )
+        xym_res['validation_time'] = td
         return extracted_models, xym_res

yangvalidator/v2/yangdumpProParser.py

@@ -19,6 +19,7 @@
 
 import logging
 import os
+from time import perf_counter
 import typing as t
 from datetime import datetime, timezone
 from subprocess import CalledProcessError, call, check_output
@@ -60,8 +61,10 @@ def parse_module(self):
         yangdump_res: t.Dict[str, t.Union[str, int]] = {'time': datetime.now(timezone.utc).isoformat()}
         ypoutfp = open(self.__yangdump_outfile, 'w+')
         ypresfp = open(self.__yangdump_resfile, 'w+')
-
+
+        t0 = perf_counter()
         status = call(self.__yangdump_command, stdout=ypoutfp, stderr=ypresfp)
+        td = perf_counter()-t0
 
         yangdump_output = yangdump_stderr = ''
         if os.path.isfile(self.__yangdump_outfile):
@@ -85,4 +88,5 @@ def parse_module(self):
         yangdump_res['version'] = self.VERSION
         yangdump_res['code'] = status
         yangdump_res['command'] = ' '.join(self.__yangdump_command)
+        yangdump_res['validation_time'] = td
         return yangdump_res

yangvalidator/v2/yanglintParser.py

@@ -19,6 +19,7 @@
 
 import logging
 import os
+from time import perf_counter
 import typing as t
 from datetime import datetime, timezone
 from subprocess import CalledProcessError, call, check_output
@@ -51,7 +52,11 @@ def parse_module(self):
 
         yresfp = open(self.__yanglint_resfile, 'w+')
         outfp = open(self.__yanglint_outfile, 'w+')
+
+        t0 = perf_counter()
         status = call(self.__yanglint_cmd, stdout=outfp, stderr=yresfp)
+        td = perf_counter()-t0
+
         self.LOG.info(f'Starting to yanglint parse use command {" ".join(self.__yanglint_cmd)}')
         yanglint_output = yanglint_stderr = ''
         if os.path.isfile(self.__yanglint_outfile):
@@ -74,5 +79,6 @@ def parse_module(self):
         yanglint_res['version'] = self.VERSION
         yanglint_res['code'] = status
         yanglint_res['command'] = ' '.join(self.__yanglint_cmd)
+        yanglint_res['validation_time'] = td
 
         return yanglint_res