From c85f7e5b7bc3cb2467a90785bcf48f9ad4144a52 Mon Sep 17 00:00:00 2001 From: Norayr Chilingarian Date: Wed, 21 Feb 2024 06:08:52 +0400 Subject: [PATCH] md5 sums are correctly retrieved from json file irrelevant of auth type being present or not, and then those are being correctly checked, md5 module added. --- GNUmakefile | 5 ++- src/vpkHttps.Mod | 52 ++++++++++++++++++++++---- src/vpkInstaller.Mod | 21 ++++++----- src/vpkJsonDepRetriever.Mod | 73 +++++++++++++++---------------------- src/vpkMD5Checker.Mod | 45 +++++++++++++++++++++++ src/vpkSyncer.Mod | 16 ++++---- src/vpkTools.Mod | 50 +++++++++++++++++++++++++ 7 files changed, 193 insertions(+), 69 deletions(-) create mode 100644 src/vpkMD5Checker.Mod diff --git a/GNUmakefile b/GNUmakefile index 860014e..0735337 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -15,8 +15,8 @@ DPD = deps ifndef DPS DPS := $(mkfile_dir_path)/$(DPD) endif -#all: get_deps build_deps buildThis -all: build_deps buildThis +all: get_deps build_deps buildThis +#all: build_deps buildThis get_deps: @echo "getting deps" @@ -60,6 +60,7 @@ buildThis: cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkStorage.Mod cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkConf.Mod cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkdepTree.Mod + cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkMD5Checker.Mod cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkHttps.Mod cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkSyncer.Mod cd $(BUILD) && $(VOC) -s $(mkfile_dir_path)/src/vpkDot.Mod diff --git a/src/vpkHttps.Mod b/src/vpkHttps.Mod index 512824c..09291ca 100644 --- a/src/vpkHttps.Mod +++ b/src/vpkHttps.Mod @@ -1,21 +1,59 @@ MODULE vpkHttps; -IMPORT Out, - List, https, - vpkdepTree; +IMPORT Out, Platform, + List, strTypes, https, + vpkdepTree, vpkTools, vpkMD5Checker, vpkEnv; PROCEDURE fetchFiles*(VAR dep: vpkdepTree.Tdep; dst: ARRAY OF CHAR); VAR - i: LONGINT; + h: https.TLSClient; + answer: strTypes.pstring; + domain, path: ARRAY 128 OF CHAR; port: ARRAY 8 OF CHAR; + filename: ARRAY 64 OF CHAR; + i, j: LONGINT; node: List.Node; + bool: BOOLEAN; + initialDst: ARRAY 512 OF CHAR; BEGIN - Out.String("fetching "); - Out.Int(dep^.rmt(vpkdepTree.RemoteHttps)^.Files.Count, 0); Out.String(" files"); Out.Ln; + COPY(dst, initialDst); + (*Out.Int(dep^.rmt(vpkdepTree.RemoteHttps)^.Files.Count, 0); Out.String(" files"); Out.Ln;*) + COPY("443", port); i := 0; REPEAT node := dep^.rmt(vpkdepTree.RemoteHttps)^.Files.Get(dep^.rmt(vpkdepTree.RemoteHttps)^.Files, i); - Out.String("file: "); Out.String(node^.obj(vpkdepTree.File)^.URI); Out.Ln; + Out.String("getting "); Out.String(node^.obj(vpkdepTree.File)^.URI); Out.Ln; + vpkTools.extractDomainFromUrl(node^.obj(vpkdepTree.File)^.URI, domain); + Out.String("connecting to "); Out.String(domain); Out.Ln; + vpkTools.extractPathFromUrl(node^.obj(vpkdepTree.File)^.URI, path); + vpkTools.extractFilenameFromUrl(node^.obj(vpkdepTree.File)^.URI, filename); + + vpkEnv.mkPkgDirPath(domain, dep^.name^, dst); + + j := Platform.Chdir(dst); + + COPY(initialDst, dst); + + h := https.Create(domain, port, path); + + IF node^.obj(vpkdepTree.File)^.auth THEN + Out.String("requires basic auth"); Out.Ln; + Out.String("username: "); Out.String(node^.obj(vpkdepTree.File)^.username); + Out.Ln; + Out.String("password: "); Out.String(node^.obj(vpkdepTree.File)^.password); + Out.Ln; + + h.appendAuthHdr(h, node^.obj(vpkdepTree.File)^.username, node^.obj(vpkdepTree.File)^.password); + + ELSE + Out.String("does not require basic auth") + END; + (* Out.String("md5: "); Out.String(node^.obj(vpkdepTree.File)^.md5); Out.Ln;*) + h.Init(h); + answer := h.Get(h); + h.Save(h); + bool := vpkMD5Checker.checkMD5(filename, node^.obj(vpkdepTree.File)^.md5); + IF bool THEN Out.String("correct!") ELSE Out.String("incorrect!"); END; Out.Ln; INC(i) UNTIL i = dep^.rmt(vpkdepTree.RemoteHttps)^.Files.Count; diff --git a/src/vpkInstaller.Mod b/src/vpkInstaller.Mod index d3fec50..e7670de 100644 --- a/src/vpkInstaller.Mod +++ b/src/vpkInstaller.Mod @@ -1,6 +1,6 @@ MODULE vpkInstaller; IMPORT In, Out, Platform, - StringList, + List, StringList, vpkResolver, vpkdepTree, vpkDot, vpkEnv, vpkJsonDepRetriever,vpkSyncer, vpkTools, vpkSettings; @@ -70,6 +70,7 @@ VAR builddir, cmd, srcPath: StringList.pstring; res: INTEGER; domain: ARRAY 256 OF CHAR; + node: List.Node; BEGIN IF prefix # "" THEN builddir := vpkEnv.mkBldDir(prefix) @@ -82,24 +83,26 @@ BEGIN REPEAT dep := vpkdepTree.Get(depTree, i); vpkJsonDepRetriever.getURIandType(dep); - Out.String("aftergetURIandType"); Out.Ln; - Out.String("got uri: "); Out.String(dep^.rmt^.URI); Out.Ln; - Out.String("got type: "); - IF dep^.Type = vpkSettings.git THEN Out.String("git"); - ELSIF dep^.Type = vpkSettings.https THEN Out.String("https") END; - Out.Ln; vpkSyncer.fetch(dep, prefix); b := FALSE; b := vpkJsonDepRetriever.getBuildInfo(dep, keys, values); IF b THEN - vpkTools.extractDomainFromUrl(dep^.rmt.URI, domain); j := 0; REPEAT - k := keys.GetString(keys, j); + IF dep^.rmt IS vpkdepTree.RemoteGit THEN + vpkTools.extractDomainFromUrl(dep^.rmt.URI, domain); + ELSIF dep^.rmt IS vpkdepTree.RemoteHttps THEN + node := dep^.rmt(vpkdepTree.RemoteHttps)^.Files.Get(dep^.rmt(vpkdepTree.RemoteHttps)^.Files, j); + vpkTools.extractDomainFromUrl(node^.obj(vpkdepTree.File)^.URI, domain) + ELSE + Out.String("WARNING: building for neither git nor https sources not supported yet"); Out.Ln; + END; +k := keys.GetString(keys, j); v := values.GetString(values, j); srcPath := vpkEnv.getSrcRelPath(dep.name^, domain, v^); cmd := vpkEnv.mkCmd(k^, srcPath^); Out.String(cmd^); Out.Ln; + Out.String("building in "); Out.String(builddir^); Out.Ln; res := Platform.Chdir(builddir^); IF res # 0 THEN Out.String("failed to change directory to "); Out.String(builddir^); Out.Ln; Out.String("this should never happen."); Out.Ln; HALT(66); diff --git a/src/vpkJsonDepRetriever.Mod b/src/vpkJsonDepRetriever.Mod index c50da71..96187c8 100644 --- a/src/vpkJsonDepRetriever.Mod +++ b/src/vpkJsonDepRetriever.Mod @@ -74,7 +74,7 @@ PROCEDURE getURIandType*(VAR d: vpkdepTree.Tdep); VAR jsonstr, errstr: strUtils.pstring; tree, singleValue, remoteValue: Json.Value; - rootObj, someObj, remoteObj, fileObj, authObj: Json.Obj; + rootObj, someObj, remoteObj, fileObj, fileObj2, authObj: Json.Obj; filesArray, fileObjValue: Json.Value; fileValue, urlValue, authTypeValue, md5Value, authCredsValue, userValue, passwordValue: Json.Value; err: ARRAY ErrmessSize OF CHAR; @@ -117,7 +117,6 @@ BEGIN IF singleValue IS Json.Obj THEN someObj := singleValue(Json.Obj); key := someObj.name; - Out.String("key: "); Out.String(key^); Out.Ln; IF someObj.value IS Json.Str THEN val := someObj.value(Json.Str).str; IF key^ = vpkSettings.rmtTypKey THEN (* type *) @@ -133,12 +132,9 @@ BEGIN IF someObj.value IS Json.Arr THEN filesArray := someObj.value(Json.Arr); - Out.String("assigned"); Out.Ln; WHILE filesArray # NIL DO IF filesArray IS Json.Arr THEN - Out.String(" files array is json.arr "); Out.Ln; fileObjValue := filesArray(Json.Arr).value; - Out.String("fileobjval := filesArray(json.obj).value"); Out.Ln; IF fileObjValue IS Json.Obj THEN fileObj := fileObjValue(Json.Obj); @@ -149,55 +145,48 @@ BEGIN COPY(vpkSettings.rmtFileURL, url^); IF Json.ObjSelect(urlValue, fileObj, url) & (urlValue IS Json.Str) THEN COPY(urlValue(Json.Str).str^, httpsFile^.URI); - Out.String("url: "); Out.String(httpsFile^.URI); Out.Ln; END; + fileObj2 := fileObj; (* extract authtype *) NEW(authType, Strings.Length(vpkSettings.rmtFileAuthType)+1); COPY(vpkSettings.rmtFileAuthType, authType^); - IF Json.ObjSelect(authTypeValue, fileObj, authType) + IF Json.ObjSelect(authTypeValue, fileObj2, authType) & (authTypeValue IS Json.Str) THEN authTypeStr := authTypeValue(Json.Str).str; - Out.String("authtype: "); Out.String(authTypeStr^); Out.Ln; - IF authTypeValue(Json.Str).str^ = vpkSettings.rmtFileAuthValBasic THEN - httpsFile^.auth := TRUE; - ELSE - httpsFile^.auth := FALSE; - END + IF authTypeValue(Json.Str).str^ = vpkSettings.rmtFileAuthValBasic THEN + httpsFile^.auth := TRUE; + (* extract auth credentials *) + NEW(authCr, Strings.Length(vpkSettings.rmtFileAuthCreds)+1); + COPY(vpkSettings.rmtFileAuthCreds, authCr^); + IF Json.ObjSelect(authCredsValue, fileObj2, authCr) & (authCredsValue IS Json.Obj) THEN + IF authCredsValue IS Json.Obj THEN + authObj := authCredsValue(Json.Obj); + NEW(user, Strings.Length(vpkSettings.rmtFileAuthUsr) + 1); + COPY(vpkSettings.rmtFileAuthUsr, user^); + IF Json.ObjSelect(userValue, authObj, user) & (userValue IS Json.Str) THEN + COPY(userValue(Json.Str).str^, httpsFile^.username); + END; + NEW(password, Strings.Length(vpkSettings.rmtFileAuthPwd) + 1); + COPY(vpkSettings.rmtFileAuthPwd, password^); + IF Json.ObjSelect(passwordValue, authObj, password) & (passwordValue IS Json.Str) THEN + COPY(passwordValue(Json.Str).str^, httpsFile^.password); + END; + END; (* if authcradsvalue is json.obj *) + END; (* if json objselect authcredsval *) + + ELSE + httpsFile^.auth := FALSE; + END END; - (* extract auth credentials *) - NEW(authCr, Strings.Length(vpkSettings.rmtFileAuthCreds)+1); - COPY(vpkSettings.rmtFileAuthCreds, authCr^); - Out.String("searching for authcreds"); Out.Ln; - IF Json.ObjSelect(authCredsValue, fileObj, authCr) & (authCredsValue IS Json.Obj) THEN - Out.String("found auth creds"); Out.Ln; - IF authCredsValue IS Json.Obj THEN - authObj := authCredsValue(Json.Obj); - Out.String("authcredsval is json.obj"); Out.Ln; - NEW(user, Strings.Length(vpkSettings.rmtFileAuthUsr) + 1); - COPY(vpkSettings.rmtFileAuthUsr, user^); - IF Json.ObjSelect(userValue, authObj, user) & (userValue IS Json.Str) THEN - COPY(userValue(Json.Str).str^, httpsFile^.username); - Out.String("user: "); Out.String(httpsFile^.username); Out.Ln; - END; - NEW(password, Strings.Length(vpkSettings.rmtFileAuthPwd) + 1); - COPY(vpkSettings.rmtFileAuthPwd, password^); - IF Json.ObjSelect(passwordValue, authObj, password) & (passwordValue IS Json.Str) THEN - COPY(passwordValue(Json.Str).str^, httpsFile^.password); - Out.String("pw: "); Out.String(httpsFile^.password); Out.Ln; - END; - END; (* if authcradsvalue is json.obj *) - END; (* if json objselect authcredsval *) - (* extract md5 *) + (* extract md5 *) NEW(md5, Strings.Length(vpkSettings.rmtFileMD5)+1); COPY(vpkSettings.rmtFileMD5, md5^); IF Json.ObjSelect(md5Value, fileObj, md5) & (md5Value IS Json.Str) THEN COPY(md5Value(Json.Str).str^, httpsFile^.md5); - Out.String("md5: "); Out.String(httpsFile^.md5); Out.Ln; + Out.String("found md5 "); Out.String(httpsFile^.md5); Out.Ln; END; httpsRemote^.Files.Append(httpsRemote^.Files, httpsFile); - Out.String("appending "); Out.String(httpsFile^.URI); Out.Ln; - Out.String("list count is :"); - Out.Int(d^.rmt(vpkdepTree.RemoteHttps)^.Files.Count, 0); Out.Ln; + (*Out.Int(d^.rmt(vpkdepTree.RemoteHttps)^.Files.Count, 0); Out.Ln;*) END; (*fileobj is json.obj *) END; (*filesarray is json arr *) filesArray := filesArray(Json.Arr).next; @@ -216,14 +205,12 @@ BEGIN IF singleValue IS Json.Obj THEN someObj := singleValue(Json.Obj); key := someObj.name; - Out.String("git key: "); Out.String(key^); Out.Ln; val := someObj.value(Json.Str).str; IF key^ = vpkSettings.rmtTreeBranchKey THEN COPY(val^, d^.rmt(vpkdepTree.RemoteGit)^.branch); END; IF key^ = vpkSettings.rmtTreeKey THEN COPY(val^, d^.rmt^.URI); - Out.String("uri found: "); Out.String(d^.rmt^.URI); Out.Ln; END; END; singleValue := someObj.next; diff --git a/src/vpkMD5Checker.Mod b/src/vpkMD5Checker.Mod new file mode 100644 index 0000000..114d23d --- /dev/null +++ b/src/vpkMD5Checker.Mod @@ -0,0 +1,45 @@ +MODULE vpkMD5Checker; +IMPORT MD5 := ethMD5, Out, Files, Strings; + +PROCEDURE checkMD5*(VAR filename, expectedMD5: ARRAY OF CHAR): BOOLEAN; +VAR + result: BOOLEAN; + F: Files.File; R: Files.Rider; + ch: CHAR; + input: POINTER TO ARRAY OF CHAR; + i, fileLength: LONGINT; + context: MD5.Context; digest: MD5.Digest; hexDigest: ARRAY 33 OF CHAR; +BEGIN + result := FALSE; + + F := Files.Old(filename); + IF F # NIL THEN + Files.Set(R, F, 0); + fileLength := Files.Length(F); + NEW(input, fileLength+1); + i := 0; + REPEAT + Files.Read(R, ch); + input^[i] := ch; + INC(i); + UNTIL R.eof; + context := MD5.New(); + MD5.WriteBytes(context, input^, LEN(input^)-1); + MD5.Close(context, digest); + MD5.ToString(digest, hexDigest); + Out.String("MD5 hash of "); Out.String(filename); Out.String(":"); Out.Ln; + Out.String(hexDigest); Out.Ln; + Out.String("exected hash:"); Out.Ln; + Out.String(expectedMD5); Out.Ln; + IF hexDigest = expectedMD5 THEN result := TRUE; END; + ELSE + Out.String("file "); Out.String(filename); Out.String("not found"); Out.Ln; + Out.String("unable to check md5 sum"); Out.Ln; + END; + RETURN result +END checkMD5; + + + + +END vpkMD5Checker. diff --git a/src/vpkSyncer.Mod b/src/vpkSyncer.Mod index 53e4df3..11142ca 100644 --- a/src/vpkSyncer.Mod +++ b/src/vpkSyncer.Mod @@ -26,18 +26,18 @@ END sync; PROCEDURE fetch*(VAR dep: vpkdepTree.Tdep; dst: ARRAY OF CHAR); VAR domain: ARRAY 64 OF CHAR; BEGIN - vpkTools.extractDomainFromUrl(dep^.rmt^.URI, domain); - IF dst = "" THEN - vpkEnv.mkdefPkgDirPath(domain, dep^.name^, dst); - ELSE - vpkEnv.mkPkgDirPath(domain, dep^.name^, dst); - END; - (* dst is changed by mkdefPkgDirPath and mkPkgdirPath *) - (*IF dep^.Type = vpkSettings.git THEN*) IF dep^.rmt # NIL THEN IF dep^.rmt IS vpkdepTree.RemoteGit THEN + vpkTools.extractDomainFromUrl(dep^.rmt^.URI, domain); + (* dst is changed by mkdefPkgDirPath and mkPkgdirPath *) + IF dst = "" THEN + vpkEnv.mkdefPkgDirPath(domain, dep^.name^, dst); + ELSE + vpkEnv.mkPkgDirPath(domain, dep^.name^, dst); + END; vpkGit.fetchRepo(dep^.rmt^.URI, dst, dep^.rmt(vpkdepTree.RemoteGit).branch); ELSIF dep^.rmt IS vpkdepTree.RemoteHttps THEN + (* full dst will be determined in vpkHttps.fetchFiles for each file *) vpkHttps.fetchFiles(dep, dst); ELSE Out.String("TODO: neither git nor https url"); Out.Ln; diff --git a/src/vpkTools.Mod b/src/vpkTools.Mod index 711ee1f..b13787b 100644 --- a/src/vpkTools.Mod +++ b/src/vpkTools.Mod @@ -1,4 +1,6 @@ MODULE vpkTools; +IMPORT Strings, strUtils; + PROCEDURE extractDomainFromUrl*(VAR url: ARRAY OF CHAR; VAR domain: ARRAY OF CHAR); VAR @@ -28,5 +30,53 @@ BEGIN domain[j] := 0X; (* Ensure the domain string is null-terminated *) END extractDomainFromUrl; +PROCEDURE extractPathFromUrl*(VAR url: ARRAY OF CHAR; VAR path: ARRAY OF CHAR); +VAR + i, j, domainEnd: INTEGER; + foundDomain: BOOLEAN; +BEGIN + i := 0; foundDomain := FALSE; + WHILE ~foundDomain & (i < LEN(url) - 1) & (url[i] # 0X) DO + IF (url[i] = ':') & (url[i + 1] = '/') & (url[i + 2] = '/') THEN + foundDomain := TRUE; + i := i + 3; + END; + INC(i); + END; + + IF foundDomain THEN + domainEnd := i; + WHILE (domainEnd < LEN(url)) & (url[domainEnd] # 0X) & (url[domainEnd] # '/') DO + INC(domainEnd); + END; + + j := 0; + WHILE (domainEnd < LEN(url)) & (url[domainEnd] # 0X) & (j < LEN(path) - 1) DO + path[j] := url[domainEnd]; + INC(j); INC(domainEnd); + END; + path[j] := 0X; + ELSE + path[0] := 0X; (* if no domain found *) + END; +END extractPathFromUrl; + +PROCEDURE extractFilenameFromUrl*(VAR url: ARRAY OF CHAR; VAR result: ARRAY OF CHAR); +VAR + lastSlashPos: INTEGER; +BEGIN + (* Find the position of the last '/' character in the URL *) + lastSlashPos := -1; + lastSlashPos := strUtils.Rpos(url, '/'); + + (* Extract the part of the URL after the last '/' to get the filename *) + IF lastSlashPos >= 0 THEN + Strings.Extract(url, lastSlashPos + 1, Strings.Length(url) - lastSlashPos - 1, result); + ELSE + (* If '/' is not found, assume the entire URL is a filename *) + COPY(url, result); + END; +END extractFilenameFromUrl; + END vpkTools.