I am using a JSON file to send data from an LDAP database on linux ADDC SAMBA for further processing. I fetch the data with a script written in python3. My problem is that some fields contain Polish characters that are encoded in unicode, for example "Bo\u017Cena \u017Ar\u00F3dlana" should be "Bożena Źródlana" . I would like the file to contain already decoded data so that I can read them without guessing what character is behind the unicode code.
I need to ask for help where in my code I should put something similar to a decoder so that the entire file is already saved as decoded and containing Polish special characters
my python3 code:
#! /usr/bin/python3
import os
import configparser
import getpass
import sys
import json
import ssl
import shutil
from ldap3 import Server, Connection, Tls, ALL_ATTRIBUTES
from datetime import date
# screen cleaner
os.system('clear')
# timestamp
current_datetime = str(date.today())
# load main config files
main_conf_file = "/tmp/ldap-searchlight/config/searchlight.conf"
config = configparser.RawConfigParser()
config.read(main_conf_file)
# variables
main_path = config['GLOBAL']['main_path']
conf_path = config['GLOBAL']['conf_path']
data_path = config['GLOBAL']['data_path']
arch_patch = config['GLOBAL']['arch_patch']
json_users_file = config['USERS']['json_users_file']
json_cmptrs_file = config['CMPTRS']['json_cmptrs_file']
# ldap variables
ldap_base_dn = config['GLOBAL']["ldap-base-dn"]
ldap_users = config['USERS']['ldap-users']
ldap_cmptrs = config['CMPTRS']['ldap_cmptrs']
user1_name = config['USERS']['user1-name']
user2_name = config['USERS']['user2-name']
user3_name = config['USERS']['user3-name']
user4_name = config['USERS']['user4-name']
user5_name = config['USERS']['user5-name']
# user's choice
print(
"Logujesz się jako:\n" +
" wybierz [ 1 ] dla " + user1_name + "\n" +
" wybierz [ 2 ] dla " + user2_name + "\n" +
" wybierz [ 3 ] dla " + user3_name + "\n" +
" wybierz [ 4 ] dla " + user4_name + "\n" +
" wybierz [ 5 ] dla " + user5_name + "\n"
)
input_name = input("WYBRANO: ")
if input_name == "1" :
user = config["USERS"]["ldap-user1"]
elif input_name == "2" :
user = config["USERS"]["ldap-user2"]
elif input_name == "3" :
user = config["USERS"]["ldap-user3"]
elif input_name == "4" :
user = config["USERS"]["ldap-user4"]
elif input_name == "5" :
user = config["USERS"]["ldap-user5"]
else:
print("Permission danied\n")
sys.exit(1)
password = getpass.getpass()
LDAP_HOST = config['GLOBAL']['ldap-host']
LDAP_USER = user +","+ ldap_users +","+ ldap_base_dn
LDAP_PASSWORD = password
tls_configuration = Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLSv1)
def ldap_server():
return Server(LDAP_HOST, use_ssl=True, tls=tls_configuration, get_info=ALL_ATTRIBUTES)
def ldap_connection():
server = ldap_server(),
return Connection(server, user=LDAP_USER,
password=LDAP_PASSWORD,
auto_bind=True)
# ldap users
LDAP_BASE_DN = ldap_users +","+ ldap_base_dn
LDAP_OBJECT_FILTER = '(objectclass=user)'
user_attr_list=[ \
'cn', \
'sn', \
'givenName', \
'instanceType', \
'whenCreated', \
'displayName', \
'uSNCreated', \
'name', \
'objectGUID', \
'badPwdCount', \
'codePage', \
'countryCode', \
'badPasswordTime', \
'lastLogoff', \
'lastLogon',\
'primaryGroupID', \
'objectSid', \
'accountExpires', \
'logonCount', \
'sAMAccountName', \
'sAMAccountType', \
'userPrincipalName', \
'objectCategory', \
'pwdLastSet', \
'userAccountControl', \
'lastLogonTimestamp', \
'whenChanged', \
'uSNChanged', \
'memberOf', \
'distinguishedName' ]
conn = ldap_connection()
conn.search(LDAP_BASE_DN, LDAP_OBJECT_FILTER, attributes=user_attr_list)
# output to json
json_users_data = main_path + data_path + json_users_file
data = json.loads(conn.response_to_json())
with open(json_users_data, 'w') as jsonfile:
json.dump(data, jsonfile)
# copy data to archive
json_users_arch = main_path + arch_patch + current_datetime + "_" + json_users_file
shutil.copy2(json_users_data, json_users_arch)
# ldap computers
LDAP_BASE_DN = ldap_cmptrs +","+ ldap_base_dn
LDAP_OBJECT_FILTER = '(objectclass=computer)'
cmptr_attr_list=[ \
'cn', \
'instanceType', \
'whenCreated', \
'uSNCreated', \
'name', \
'objectGUID', \
'badPwdCount', \
'codePage', \
'countryCode', \
'badPasswordTime', \
'lastLogoff', \
'lastLogon',\
'primaryGroupID', \
'accountExpires', \
'logonCount', \
'sAMAccountName', \
'sAMAccountType', \
'objectCategory', \
'pwdLastSet', \
'userAccountControl', \
'lastLogonTimestamp', \
'whenChanged', \
'uSNChanged', \
'dNSHostName', \
'isCriticalSystemObject', \
'msDS-SupportedEncryptionTypes', \
'operatingSystem', \
'operatingSystemVersion', \
'servicePrincipalName', \
'distinguishedName' ]
conn = ldap_connection()
conn.search(LDAP_BASE_DN, LDAP_OBJECT_FILTER, attributes=cmptr_attr_list)
# output to json
json_cmptrs_data = main_path + data_path + json_cmptrs_file
data = json.loads(conn.response_to_json())
with open(json_cmptrs_data, 'w') as jsonfile:
json.dump(data, jsonfile)
# copy data
json_cmptrs_arch = main_path + arch_patch + current_datetime + "_" + json_cmptrs_file
shutil.copy2(json_cmptrs_data, json_cmptrs_arch)
print("USERS:")
print("Data file created at: " + json_users_data)
print("Archive file created at: " + json_users_arch)
print("------------------------------------------------------------------------------")
print("COMPUTERS")
print("Data file created at: " + json_cmptrs_data)
print("Archive file created at: " + json_cmptrs_arch)
sys.exit(0)
# exit(0) -> OK
# exit(1) -> FAULT
my jsons output looks:
{"entries": [
{"attributes":
{
"accountExpires": ["9223372036854775807"],
"badPasswordTime": [],
"badPwdCount": [],
"cn": ["Bo\u017Cena \u017Ar\u00F3dlana"],
"codePage": ["0"],
"countryCode": ["0"],
"displayName": ["Bo\u017Cena \u017Ar\u00F3dlana"],
"distinguishedName": ["CN=Bo\u017Cena \u017Ar\u00F3dlana,OU=FE,OU=Users,OU=UNIVERSUM,DC=universum,DC=local"],
"givenName": ["Bo\u017Cena"],
"instanceType": ["4"],
"lastLogoff": [],
"lastLogon": [],
"lastLogonTimestamp": ["132978476924537530"],
"logonCount": [],
"memberOf": [],
"name": ["Bo\u017Cena \u017Ar\u00F3dlana"],
"objectCategory": ["CN=Person,CN=Schema,CN=Configuration,DC=universum,DC=local"],
"objectGUID": [
{
"encoded": "AFvzBO0T+Ey9TL3RHGtghQ==",
"encoding": "base64"
}
],
"objectSid": [
{
"encoded": "AQUAAAAAAAUVAAAA6TO9FZD9W8QoWlFDIE8AAA==",
"encoding": "base64"
}
],
"primaryGroupID": ["513"],
"pwdLastSet": ["132979783101549910"],
"sAMAccountName": ["pjarmolowicz"],
"sAMAccountType": ["805306368"],
"sn": ["\u017Ar\u00F3dlana"],
"uSNChanged": ["4986"],
"uSNCreated": ["4986"],
"userAccountControl": ["512"],
"userPrincipalName": ["bzrodlana#universum.local"],
"whenChanged": ["20220525185150.0Z"],
"whenCreated": ["20211125124337.0Z"]},
"dn": "CN=Bo\u017Cena \u017Ar\u00F3dlana,OU=FE,OU=Users,OU=UNIVERSUM,DC=universum,DC=local"
},
{"attributes": {
"accountExpires": ["9223372036854775807"],
"badPasswordTime": ["133128872888506790"],
"badPwdCount": ["0"],
"cn": ["Jan Kowalski"],
"codePage": ["0"],
"countryCode": ["0"],
"displayName": ["Jan Kowalski"],
"distinguishedName": ["CN=Jan Kowalski,OU=RR-32,OU=RR,OU=Users,OU=UNIVERSUM,DC=universum,DC=local"],
"givenName": ["Jan"],
"instanceType": ["4"],
"lastLogoff": [],
"lastLogon": ["133129921828641420"],
"lastLogonTimestamp": ["133125345565644950"],
"logonCount": ["55"],
"memberOf": [],
"name": ["Jan Kowalski"],
"objectCategory": ["CN=Person,CN=Schema,CN=Configuration,DC=universum,DC=local"],
"objectGUID": [
{
"encoded": "AScnTASpKUun4oadMC5Qxg==",
"encoding": "base64"
}
],
"objectSid": [
{
"encoded": "AQUAAAAAAAUVAAAA6TO9FZD9W8QoWlFDngQAAA==",
"encoding": "base64"
}
],
"primaryGroupID": ["513"],
"pwdLastSet": ["131577266641617910"],
"sAMAccountName": ["jkowalski"],
"sAMAccountType": ["805306368"],
"sn": ["Kowalski"],
"uSNChanged": ["149609"],
"uSNCreated": ["5397"],
"userAccountControl": ["512"],
"userPrincipalName": ["jkowalski#universum.local"],
"whenChanged": ["20221110061556.0Z"],
"whenCreated": ["20130610115016.0Z"],
"dn": "CN=Jan Kowalski,OU=RR-32,OU=RR,OU=Users,OU=UNIVERSUM,DC=universum,DC=local"
}
]
}
Use the following to suppress Unicode escape codes and write the data UTF-8-encoded to support non-ASCII characters.
with open(json_cmptrs_data, 'w', encoding='utf8') as jsonfile:
json.dump(data, jsonfile, ensure_ascii=False)
Working example:
import json
data = {"cn": ["Bo\u017Cena \u017Ar\u00F3dlana"]}
with open('output.json', 'w', encoding='utf8') as file:
json.dump(data, file, ensure_ascii=False)
output.csv (UTF-8-encoded):
{"cn": ["Bożena źródlana"]}
I am trying to modify the code below to check the header URLs from a list
Usage
$ python securityheaders.py --max-redirects 5 https://secfault.fi
Output
Header 'x-xss-protection' is missing ... [ WARN ]
Header 'x-content-type-options' is missing ... [ WARN ]
Header 'content-security-policy' is missing ... [ WARN ]
Header 'x-powered-by' is missing ... [ OK ]
Header 'x-frame-options' contains value 'DENY' ... [ OK ]
Header 'strict-transport-security' contains value 'max-age=63072000' ... [ OK ]
Header 'access-control-allow-origin' is missing ... [ OK ]
Header 'server' contains value 'nginx/1.10.1' ... [ WARN ]
HTTPS supported ... [ OK ]
HTTPS valid certificate ... [ OK ]
HTTP -> HTTPS redirect ... [ OK ]
My question is the following:
Instead of passing one URL as argument like the usage above, is it possible to pass a list of URL to check the header and then save the result into the excel file like below.
Which part of the code should I modify?
Any help would be much appreciated
Thank you for reading
Desire ouput
Code
Source code
import http.client
import argparse
import socket
import ssl
import sys
import re
from urllib.parse import urlparse
class SecurityHeaders():
def __init__(self):
pass
def evaluate_warn(self, header, contents):
""" Risk evaluation function.
Set header warning flag (1/0) according to its contents.
Args:
header (str): HTTP header name in lower-case
contents (str): Header contents (value)
"""
warn = 1
if header == 'x-frame-options':
if contents.lower() in ['deny', 'sameorigin']:
warn = 0
else:
warn = 1
if header == 'strict-transport-security':
warn = 0
""" Evaluating the warn of CSP contents may be a bit more tricky.
For now, just disable the warn if the header is defined
"""
if header == 'content-security-policy':
warn = 0
""" Raise the warn flag, if cross domain requests are allowed from any
origin """
if header == 'access-control-allow-origin':
if contents == '*':
warn = 1
else:
warn = 0
if header.lower() == 'x-xss-protection':
if contents.lower() in ['1', '1; mode=block']:
warn = 0
else:
warn = 1
if header == 'x-content-type-options':
if contents.lower() == 'nosniff':
warn = 0
else:
warn =1
""" Enable warning if backend version information is disclosed """
if header == 'x-powered-by' or header == 'server':
if len(contents) > 1:
warn = 1
else:
warn = 0
return {'defined': True, 'warn': warn, 'contents': contents}
def test_https(self, url):
parsed = urlparse(url)
protocol = parsed[0]
hostname = parsed[1]
path = parsed[2]
sslerror = False
conn = http.client.HTTPSConnection(hostname, context = ssl.create_default_context() )
try:
conn.request('GET', '/')
res = conn.getresponse()
except socket.gaierror:
return {'supported': False, 'certvalid': False}
except ssl.CertificateError:
return {'supported': True, 'certvalid': False}
except:
sslerror = True
# if tls connection fails for unexcepted error, retry without verifying cert
if sslerror:
conn = http.client.HTTPSConnection(hostname, timeout=5, context = ssl._create_stdlib_context() )
try:
conn.request('GET', '/')
res = conn.getresponse()
return {'supported': True, 'certvalid': False}
except:
return {'supported': False, 'certvalid': False}
return {'supported': True, 'certvalid': True}
def test_http_to_https(self, url, follow_redirects = 5):
parsed = urlparse(url)
protocol = parsed[0]
hostname = parsed[1]
path = parsed[2]
if not protocol:
protocol = 'http' # default to http if protocl scheme not specified
if protocol == 'https' and follow_redirects != 5:
return True
elif protocol == 'https' and follow_redirects == 5:
protocol = 'http'
if (protocol == 'http'):
conn = http.client.HTTPConnection(hostname)
try:
conn.request('HEAD', path)
res = conn.getresponse()
headers = res.getheaders()
except socket.gaierror:
print('HTTP request failed')
return False
""" Follow redirect """
if (res.status >= 300 and res.status < 400 and follow_redirects > 0):
for header in headers:
if (header[0].lower() == 'location'):
return self.test_http_to_https(header[1], follow_redirects - 1)
return False
def check_headers(self, url, follow_redirects = 0):
""" Make the HTTP request and check if any of the pre-defined
headers exists.
Args:
url (str): Target URL in format: scheme://hostname/path/to/file
follow_redirects (Optional[str]): How deep we follow the redirects,
value 0 disables redirects.
"""
""" Default return array """
retval = {
'x-frame-options': {'defined': False, 'warn': 1, 'contents': '' },
'strict-transport-security': {'defined': False, 'warn': 1, 'contents': ''},
'access-control-allow-origin': {'defined': False, 'warn': 0, 'contents': ''},
'content-security-policy': {'defined': False, 'warn': 1, 'contents': ''},
'x-xss-protection': {'defined': False, 'warn': 1, 'contents': ''},
'x-content-type-options': {'defined': False, 'warn': 1, 'contents': ''},
'x-powered-by': {'defined': False, 'warn': 0, 'contents': ''},
'server': {'defined': False, 'warn': 0, 'contents': ''}
}
parsed = urlparse(url)
protocol = parsed[0]
hostname = parsed[1]
path = parsed[2]
if (protocol == 'http'):
conn = http.client.HTTPConnection(hostname)
elif (protocol == 'https'):
# on error, retry without verifying cert
# in this context, we're not really interested in cert validity
ctx = ssl._create_stdlib_context()
conn = http.client.HTTPSConnection(hostname, context = ctx )
else:
""" Unknown protocol scheme """
return {}
try:
conn.request('HEAD', path)
res = conn.getresponse()
headers = res.getheaders()
except socket.gaierror:
print('HTTP request failed')
return False
""" Follow redirect """
if (res.status >= 300 and res.status < 400 and follow_redirects > 0):
for header in headers:
if (header[0].lower() == 'location'):
redirect_url = header[1]
if not re.match('^https?://', redirect_url):
redirect_url = protocol + '://' + hostname + redirect_url
return self.check_headers(redirect_url, follow_redirects - 1)
""" Loop through headers and evaluate the risk """
for header in headers:
#set to lowercase before the check
headerAct = header[0].lower()
if (headerAct in retval):
retval[headerAct] = self.evaluate_warn(headerAct, header[1])
return retval
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Check HTTP security headers', \
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('url', metavar='URL', type=str, help='Target URL')
parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int, help='Max redirects, set 0 to disable')
args = parser.parse_args()
url = args.url
redirects = args.max_redirects
foo = SecurityHeaders()
parsed = urlparse(url)
if not parsed.scheme:
url = 'http://' + url # default to http if scheme not provided
headers = foo.check_headers(url, redirects)
if not headers:
print ("Failed to fetch headers, exiting...")
sys.exit(1)
okColor = '\033[92m'
warnColor = '\033[93m'
endColor = '\033[0m'
for header, value in headers.items():
if value['warn'] == 1:
if value['defined'] == False:
print('Header \'' + header + '\' is missing ... [ ' + warnColor + 'WARN' + endColor + ' ]')
else:
print('Header \'' + header + '\' contains value \'' + value['contents'] + '\'' + \
' ... [ ' + warnColor + 'WARN' + endColor + ' ]')
elif value['warn'] == 0:
if value['defined'] == False:
print('Header \'' + header + '\' is missing ... [ ' + okColor + 'OK' + endColor +' ]')
else:
print('Header \'' + header + '\' contains value \'' + value['contents'] + '\'' + \
' ... [ ' + okColor + 'OK' + endColor + ' ]')
https = foo.test_https(url)
if https['supported']:
print('HTTPS supported ... [ ' + okColor + 'OK' + endColor + ' ]')
else:
print('HTTPS supported ... [ ' + warnColor + 'FAIL' + endColor + ' ]')
if https['certvalid']:
print('HTTPS valid certificate ... [ ' + okColor + 'OK' + endColor + ' ]')
else:
print('HTTPS valid certificate ... [ ' + warnColor + 'FAIL' + endColor + ' ]')
if foo.test_http_to_https(url, 5):
print('HTTP -> HTTPS redirect ... [ ' + okColor + 'OK' + endColor + ' ]')
else:
print('HTTP -> HTTPS redirect ... [ ' + warnColor + 'FAIL' + endColor + ' ]')
The generated Webapp "crashes" while bootstrapping the backend, because Liquibase won't be able to reach a consistent state. Crash means here, the webapp itself runs, but it won't be usable because the backend cannot authorize you. I read other posts on SO regarding Liquibase and no suggestion works.
Performed steps:
visit Jhipster Online to generate a vanilla Webapp as starting point.
download and extract the ZIP file to a directory of my choice
chmod +x ./mvnw
npm i
run ./mvnw -P-webapp
System Info
Ubuntu 20.04.03 LTS
node v14.16.0
javac openJDK 11.0.11
java:
openjdk version "11.0.11" 2021-04-20
OpenJDK Runtime Environment (build 11.0.11+9-Ubuntu-0ubuntu2.20.04)
OpenJDK 64-Bit Server VM (build 11.0.11+9-Ubuntu-0ubuntu2.20.04, mixed mode, sharing)
My steps to solve the issue
create new directories
switch node versions using nvm use --lts or i.e. nvm use [version no.]
run jhipster and select some stuff on each project
run ./mvnw -P-webapp --> "watch LiqourBase burn :-("
alternatively change java-version to i.e. 16.x
reboots
cursing, repeating upper steps with different selections
I don't know why it stops working. I observed the issue with some other recently made projects using Jhipster, too. What is it? Do I have to wipe complete .m2 simply that way?
The Log:
2021-11-09 14:22:14.210 DEBUG 22360 --- [kground-preinit] org.jboss.logging : Logging Provider: org.jboss.logging.Log4j2LoggerProvider
2021-11-09 14:22:14.253 INFO 22360 --- [ restartedMain] dev.semo.some.Webapp : Starting Webapp84 using Java 11.0.11 on computer with PID 22360 (/home/semo/dev/work/playground/target/classes started by semo in /home/semo/dev/work/playground)
2021-11-09 14:22:14.254 DEBUG 22360 --- [ restartedMain] dev.semo.some.Webapp : Running with Spring Boot v2.5.5, Spring v5.3.10
2021-11-09 14:22:14.255 INFO 22360 --- [ restartedMain] dev.semo.some.Webapp : The following profiles are active: dev,api-docs
[...omitted...]
2021-11-09 14:22:18.725 DEBUG 22360 --- [ restartedMain] d.b.semo.some.config.AsyncConfiguration : Creating Async Task Executor
2021-11-09 14:22:18.853 DEBUG 22360 --- [ restartedMain] d.b.s.some.config.LiquibaseConfiguration : Configuring Liquibase
2021-11-09 14:22:19.076 WARN 22360 --- [ebapp-84-task-1] t.j.c.liquibase.AsyncSpringLiquibase : Starting Liquibase asynchronously, your database might not be ready at startup!
2021-11-09 14:22:20.770 ERROR 22360 --- [ebapp-84-task-1] t.j.c.liquibase.AsyncSpringLiquibase : Liquibase could not start correctly, your database is NOT ready: liquibase.exception.MigrationFailedException: Migration failed for change set config/liquibase/changelog/00000000000000_initial_schema.xml::00000000000001::jhipster:
Reason: liquibase.exception.DatabaseException: Syntax Fehler in SQL Befehl "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"
Syntax error in SQL statement "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"; SQL statement:
IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON [42000-200] [Failed SQL: (42000) IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON]
liquibase.exception.LiquibaseException: liquibase.exception.MigrationFailedException: Migration failed for change set config/liquibase/changelog/00000000000000_initial_schema.xml::00000000000001::jhipster:
Reason: liquibase.exception.DatabaseException: Syntax Fehler in SQL Befehl "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"
Syntax error in SQL statement "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"; SQL statement:
IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON [42000-200] [Failed SQL: (42000) IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON]
at liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:124)
at liquibase.Liquibase.lambda$null$0(Liquibase.java:265)
at liquibase.Scope.lambda$child$0(Scope.java:177)
at liquibase.Scope.child(Scope.java:186)
at liquibase.Scope.child(Scope.java:176)
at liquibase.Scope.child(Scope.java:155)
at liquibase.Scope.child(Scope.java:239)
at liquibase.Liquibase.lambda$update$1(Liquibase.java:264)
at liquibase.Scope.lambda$child$0(Scope.java:177)
at liquibase.Scope.child(Scope.java:186)
at liquibase.Scope.child(Scope.java:176)
at liquibase.Scope.child(Scope.java:155)
at liquibase.Liquibase.runInScope(Liquibase.java:2404)
at liquibase.Liquibase.update(Liquibase.java:211)
at liquibase.Liquibase.update(Liquibase.java:197)
at liquibase.integration.spring.SpringLiquibase.performUpdate(SpringLiquibase.java:314)
at liquibase.integration.spring.SpringLiquibase.afterPropertiesSet(SpringLiquibase.java:269)
at org.springframework.boot.autoconfigure.liquibase.DataSourceClosingSpringLiquibase.afterPropertiesSet(DataSourceClosingSpringLiquibase.java:46)
at tech.jhipster.config.liquibase.AsyncSpringLiquibase.initDb(AsyncSpringLiquibase.java:118)
at tech.jhipster.config.liquibase.AsyncSpringLiquibase.lambda$afterPropertiesSet$0(AsyncSpringLiquibase.java:93)
at tech.jhipster.async.ExceptionHandlingAsyncTaskExecutor.lambda$createWrappedRunnable$1(ExceptionHandlingAsyncTaskExecutor.java:78)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: liquibase.exception.MigrationFailedException: Migration failed for change set config/liquibase/changelog/00000000000000_initial_schema.xml::00000000000001::jhipster:
Reason: liquibase.exception.DatabaseException: Syntax Fehler in SQL Befehl "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"
Syntax error in SQL statement "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"; SQL statement:
IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON [42000-200] [Failed SQL: (42000) IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON]
at liquibase.changelog.ChangeSet.execute(ChangeSet.java:695)
at liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:49)
at liquibase.changelog.ChangeLogIterator$2.lambda$null$0(ChangeLogIterator.java:111)
at liquibase.Scope.lambda$child$0(Scope.java:177)
at liquibase.Scope.child(Scope.java:186)
at liquibase.Scope.child(Scope.java:176)
at liquibase.Scope.child(Scope.java:155)
at liquibase.changelog.ChangeLogIterator$2.lambda$run$1(ChangeLogIterator.java:110)
at liquibase.Scope.lambda$child$0(Scope.java:177)
at liquibase.Scope.child(Scope.java:186)
at liquibase.Scope.child(Scope.java:176)
at liquibase.Scope.child(Scope.java:155)
at liquibase.Scope.child(Scope.java:239)
at liquibase.changelog.ChangeLogIterator$2.run(ChangeLogIterator.java:94)
at liquibase.Scope.lambda$child$0(Scope.java:177)
at liquibase.Scope.child(Scope.java:186)
at liquibase.Scope.child(Scope.java:176)
at liquibase.Scope.child(Scope.java:155)
at liquibase.Scope.child(Scope.java:239)
at liquibase.Scope.child(Scope.java:243)
at liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:66)
... 23 common frames omitted
Caused by: liquibase.exception.DatabaseException: Syntax Fehler in SQL Befehl "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"
Syntax error in SQL statement "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"; SQL statement:
IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON [42000-200] [Failed SQL: (42000) IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON]
at liquibase.executor.jvm.JdbcExecutor$ExecuteStatementCallback.doInStatement(JdbcExecutor.java:393)
at liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:82)
at liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:150)
at liquibase.database.AbstractJdbcDatabase.execute(AbstractJdbcDatabase.java:1279)
at liquibase.database.AbstractJdbcDatabase.executeStatements(AbstractJdbcDatabase.java:1261)
at liquibase.changelog.ChangeSet.execute(ChangeSet.java:660)
... 43 common frames omitted
Caused by: org.h2.jdbc.JdbcSQLSyntaxErrorException: Syntax Fehler in SQL Befehl "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"
Syntax error in SQL statement "IF[*] EXISTS(SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'jhi_user'
AND TABLE_SCHEMA = 'PUBLIC'
AND COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.JHI_USER ON"; SQL statement:
IF EXISTS(select TABLE_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = 'jhi_user'
and TABLE_SCHEMA = 'PUBLIC'
and COLUMNPROPERTY(object_id(TABLE_SCHEMA + '.' + TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1)
SET IDENTITY_INSERT PUBLIC.jhi_user ON [42000-200]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:453)
at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
at org.h2.message.DbException.get(DbException.java:205)
at org.h2.message.DbException.get(DbException.java:181)
at org.h2.message.DbException.getSyntaxError(DbException.java:229)
at org.h2.command.Parser.getSyntaxError(Parser.java:1051)
at org.h2.command.Parser.parsePrepared(Parser.java:1013)
at org.h2.command.Parser.parse(Parser.java:843)
at org.h2.command.Parser.parse(Parser.java:819)
at org.h2.command.Parser.prepareCommand(Parser.java:738)
at org.h2.engine.Session.prepareLocal(Session.java:657)
at org.h2.engine.Session.prepareCommand(Session.java:595)
at org.h2.jdbc.JdbcConnection.prepareCommand(JdbcConnection.java:1235)
at org.h2.jdbc.JdbcStatement.executeInternal(JdbcStatement.java:212)
at org.h2.jdbc.JdbcStatement.execute(JdbcStatement.java:201)
at com.zaxxer.hikari.pool.ProxyStatement.execute(ProxyStatement.java:94)
at com.zaxxer.hikari.pool.HikariProxyStatement.execute(HikariProxyStatement.java)
at liquibase.executor.jvm.JdbcExecutor$ExecuteStatementCallback.doInStatement(JdbcExecutor.java:389)
... 48 common frames omitted
2021-11-09 14:22:21.028 DEBUG 22360 --- [ restartedMain] d.b.semo.some.security.jwt.TokenProvider : Using a Base64-encoded JWT secret key
2021-11-09 14:22:21.940 INFO 22360 --- [ restartedMain] o.a.k.clients.producer.ProducerConfig : ProducerConfig values:
[...omitted...]
2021-11-09 14:22:22.057 INFO 22360 --- [ restartedMain] o.a.kafka.common.utils.AppInfoParser : Kafka version: 2.7.1
2021-11-09 14:22:22.057 INFO 22360 --- [ restartedMain] o.a.kafka.common.utils.AppInfoParser : Kafka commitId: 61dbce85d0d41457
2021-11-09 14:22:22.058 INFO 22360 --- [ restartedMain] o.a.kafka.common.utils.AppInfoParser : Kafka startTimeMs: 1636464142055
2021-11-09 14:22:22.072 WARN 22360 --- [ad | producer-1] org.apache.kafka.clients.NetworkClient : [Producer clientId=producer-1] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
[... omitted ...]
2021-11-09 14:22:22.910 DEBUG 22360 --- [ restartedMain] t.j.c.apidoc.SpringfoxAutoConfiguration : Starting OpenAPI docs
2021-11-09 14:22:22.928 DEBUG 22360 --- [ restartedMain] t.j.c.apidoc.SpringfoxAutoConfiguration : Started OpenAPI docs in 18 ms
2021-11-09 14:22:22.953 DEBUG 22360 --- [ restartedMain] d.b.s.kea.config.DatabaseConfiguration : H2 database is available on port 18501
2021-11-09 14:22:23.795 INFO 22360 --- [ restartedMain] org.jboss.threads : JBoss Threads version 3.1.0.Final
2021-11-09 14:22:24.112 INFO 22360 --- [ restartedMain] dev.semo.some.Webapp : Started Webapp84 in 10.846 seconds (JVM running for 11.77)
2021-11-09 14:22:24.116 INFO 22360 --- [ restartedMain] dev.semo.some.Webapp :
----------------------------------------------------------
Application 'webapp84' is running! Access URLs:
Local: http://localhost:8501/
External: http://127.0.1.1:8501/
Profile(s): [dev, api-docs]
----------------------------------------------------------
I am trying to convert a d3 svg image to an image based on the following tutorial: https://github.com/hugolpz/svgcreator.node.js
I install the following statements:
sudo npm install -g jsdom d3js
npm install jsdom d3js
node svgcreator.node.js > out.svg
I used the following code
var jsdom = require('jsdom');
jsdom.env(
"<html><body></body></html>", // CREATE DOM HOOK
[ 'http://d3js.org/d3.v3.min.js', // JS DEPENDENCIES online ...
'js/d3.v3.min.js' ], // ... & local-offline
function (err, window) {
// D3JS CODE * * * * * * * * * * * * * * * * * * * * * * * *
var svg = window.d3.select("body")
.append("svg")
.attr("width", 100)
.attr("height", 100);
svg.append("rect")
.attr("id", "rect1")
.attr("x", 10)
.attr("y", 10)
.attr("width", 80)
.attr("height", 80)
.style("fill", "green");
// END (D3JS) * * * * * * * * * * * * * * * * * * * * * * * *
//PRINTING OUT SELECTION
console.log( window.d3.select("body").html() );
} // end function
);
When i execute node svgcreator.node.js > out.svg i get the following error
TypeError: jsdom.env is not a function
at Object.<anonymous> (C:\Users\ErikvanderHoeven\Documents\git\svgcreator.node.js\svgcreator.node.js:5:7)
?[90m at Module._compile (internal/modules/cjs/loader.js:1063:30)?[39m
?[90m at Object.Module._extensions..js (internal/modules/cjs/loader.js:1092:10)?[39m
?[90m at Module.load (internal/modules/cjs/loader.js:928:32)?[39m
?[90m at Function.Module._load (internal/modules/cjs/loader.js:769:14)?[39m
?[90m at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:72:12)?[39m
?[90m at internal/main/run_main_module.js:17:47?[39
It seems you have a dependency version issue - the code in the Github project is several years old and the instructions are loading much later versions of the dependencies than when the example code was written.
I get an npm error trying to install d3js and jsdom is on v16.4. If I run:
npm install jsdom d3 --save
My package.json is:
{
"name": "test-jsdom",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"d3": "^6.5.0",
"jsdom": "^16.4.0"
}
}
Github user dam1r89 shows a way to do what you are trying on later versions in this gist.
My take on it:
const d3 = require("d3");
const fs = require("fs");
const {JSDOM} = require("jsdom");
// init d3 - https://gist.github.com/tomgp/c99a699587b5c5465228
const minHtml = "<html><head></head><body></body></html>";
const dom = new JSDOM(`${minHtml}`, { pretendToBeVisual: true });
const window = dom.window;
window.d3 = d3.select(window.document);
// D3JS CODE * * * * * * * * * * * * * * * * * * * * * * * *
var svg = window.d3.select("body")
.append("svg")
.attr("width", 100)
.attr("height", 100);
svg.append("rect")
.attr("id", "rect1")
.attr("x", 10)
.attr("y", 10)
.attr("width", 80)
.attr("height", 80)
.style("fill", "green");
// END (D3JS) * * * * * * * * * * * * * * * * * * * * * * * *
console.log( window.d3.select("body").html() );
Which outputs:
<svg width="100" height="100"><rect id="rect1" x="10" y="10" width="80" height="80" style="fill: green;"></rect></svg>
Latest Jenkins used.
edit: pastebin of full Java exception - https://pastebin.com/zZDNj18E
Goal: loop through all nodes, check for offline, email alert for each offline node.
(tried emailext alerts as well, could not use "offline")
Failed: My jenkinsfile runs perfectly with no email.
With email in the for loop or separately defined in a function, the job crashes after the first email is sent.
[Pipeline] End of Pipeline an exception which occurred: in field hudson.model.Slave.launcher in object hudson.slaves.DumbSlave#ae938e61 .... and many more
My jenkinsfile:
pipeline {
agent{
label 'master'
}
options {
// Enable timestamps in log
timestamps()
skipDefaultCheckout()
timeout(time: 4, unit: 'MINUTES')
}
stages {
stage('Monitor') {
steps{
script{
def offlineSlaves = []
for (aSlave in hudson.model.Hudson.instance.slaves) {
def thisSlave = aSlave.name
echo 'Name: ' + thisSlave + ' is being checked.'
if ( aSlave.getComputer().isOffline().toString() == 'true') {
slaveState = 'OFFLINE'
echo 'Name: ' + thisSlave + ' is ' + slaveState + ' !'
emailext (
mimeType: 'text/html',
body: "${env.JOB_NAME} found an OFFLINE node: ${name} ",
subject: "Jenkins ERROR: Build Node ${name} is OFFLINE " ,
to: 'jfisher#xxx')
}
}
}
}
}
}
post {
failure {
emailext (
body: 'Monitor Nodes Jenkins Job failed !',
presendScript: '$DEFAULT_PRESEND_SCRIPT',
recipientProviders: [requestor(),culprits()],
subject: 'Monitor Nodes Jenkins Failed',
to: 'jfisher#intouchhealth.com')
}
}
}
The problem with this code is the getComputer() part. In the pipeline you should only use Serializable and the SlaveComputer returned from getComputer() isn't.
https://javadoc.jenkins.io/hudson/slaves/SlaveComputer.html
What you should do is move this part to a function annotated with NonCPS
#NonCPS
def shallTrigger() {
for (aSlave in hudson.model.Hudson.instance.slaves) {
def thisSlave = aSlave.name
echo 'Name: ' + thisSlave + ' is being checked.'
if ( aSlave.getComputer().isOffline().toString() == 'true') {
slaveState = 'OFFLINE'
echo 'Name: ' + thisSlave + ' is ' + slaveState + ' !'
emailext (
mimeType: 'text/html',
body: "${env.JOB_NAME} found an OFFLINE node: ${name} ",
subject: "Jenkins ERROR: Build Node ${name} is OFFLINE " ,
to: 'jfisher#xxx')
}
}
}