mirrors/pocketlang
1
0
Fork 0
mirror of https://github.com/zekexiao/pocketlang.git synced 2025-02-06 04:37:47 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

import statements are refactored.

Browse Source 
- all import statement (native or script file) have the same syntax
- allow relative (including parent directory) imports
- cyclic imports are handled by caching the scripts
- `import foo` can potentially import `<searchpath>/foo/_init.pk`
- * import are not supported anymore
This commit is contained in:
Thakee Nathees 2022-05-06 08:16:40 +05:30
parent 9dc2a99c3f
commit 168f365cde
17 changed files with 437 additions and 512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
cli/modules/std_path.c
View File

@ -12,19 +12,31 @@
// Refactor the bellow macro includes. // Refactor the bellow macro includes.
#include "thirdparty/cwalk/cwalk.h" #include "thirdparty/cwalk/cwalk.h"
#if defined(_WIN32) && (defined(_MSC_VER) || defined(__TINYC__))
#include "thirdparty/dirent/dirent.h" #include <errno.h>
#else #include <sys/stat.h>
#include <dirent.h>
#endif
#if defined(_WIN32) #if defined(_WIN32)
#include <windows.h> #include <windows.h>
#endif
#if defined(_MSC_VER) || (defined(_WIN32) && defined(__TINYC__))
#include "thirdparty/dirent/dirent.h"
#include <direct.h> #include <direct.h>
#define get_cwd _getcwd #include <io.h>
// access() function flag defines for windows.
// Reference :https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/access-waccess?view=msvc-170#remarks
#define F_OK 0
#define W_OK 2
#define R_OK 4
#define access _access
#define getcwd _getcwd
#else #else
#include <dirent.h>
#include <unistd.h> #include <unistd.h>
#define get_cwd getcwd
#endif #endif
// The maximum path size that pocketlang's default import system supports // The maximum path size that pocketlang's default import system supports
@ -39,12 +51,76 @@
#define MAX_JOIN_PATHS 8 #define MAX_JOIN_PATHS 8
static inline size_t pathAbs(const char* path, char* buff, size_t buff_size); static inline size_t pathAbs(const char* path, char* buff, size_t buff_size);
static inline bool pathIsFileExists(const char* path); static inline bool pathIsFile(const char* path);
static inline bool pathIsDir(const char* path);
/*****************************************************************************/ /*****************************************************************************/
/* PATH SHARED FUNCTIONS */ /* PATH SHARED FUNCTIONS */
/*****************************************************************************/ /*****************************************************************************/
// check if path + ext exists. If not return 0, otherwise path + ext will
// written to the buffer and return the total length.
//
// [path] and [buff] should be char array with size of FILENAME_MAX. This
// function will write path + ext to the buff. If the path exists it'll return
// true.
static inline size_t checkImportExists(char* path, const char* ext,
char* buff) {
size_t path_size = strlen(path);
size_t ext_size = strlen(ext);
// If the path size is too large we're bailing out.
if (path_size + ext_size + 1 >= FILENAME_MAX) return 0;
// Now we're safe to use strcpy.
strcpy(buff, path);
strcpy(buff + path_size, ext);
if (!pathIsFile(buff)) return 0;
return path_size + ext_size;
}
// Try all import paths by appending pocketlang supported extensions to the
// path (ex: path + ".pk", path + ".dll", path + ".so", path + "/_init.pk", ...
// if such a path exists it'll allocate a pocketlang string and return it.
//
// Note that [path] and [buff] should be char array with size of FILENAME_MAX.
// The buffer will be used as a working memory.
static char* tryImportPaths(PKVM* vm, char* path, char* buff) {
size_t path_size = 0;
// FIXME: review this code.
do {
if ((path_size = checkImportExists(path, "", buff)) != 0) {
break;
} else if ((path_size = checkImportExists(path, ".pk", buff)) != 0) {
break;
} else if ((path_size = checkImportExists(path, "/_init.pk", buff)) != 0) {
break;
}
} while (false);
char* ret = NULL;
if (path_size != 0) {
ret = pkAllocString(vm, path_size + 1);
memcpy(ret, buff, path_size + 1);
}
return ret;
}
// replace all the '\\' with '/' to make all the seperator in a path is the
// same this is only used in import path system to make the path of a module
// unique (otherwise same path with different seperator makes them different).
void pathFixWindowsSeperator(char* buff) {
ASSERT(buff, OOPS);
while (*buff != '\0') {
if (*buff == '\\') *buff = '/';
buff++;
}
}
// Implementation of pocketlang import path resolving function. // Implementation of pocketlang import path resolving function.
char* pathResolveImport(PKVM* vm, const char* from, const char* path) { char* pathResolveImport(PKVM* vm, const char* from, const char* path) {
@ -52,14 +128,15 @@ char* pathResolveImport(PKVM* vm, const char* from, const char* path) {
char buff1[FILENAME_MAX]; char buff1[FILENAME_MAX];
char buff2[FILENAME_MAX]; char buff2[FILENAME_MAX];
// If the path is absolute, Just normalize and return it. // If the path is absolute, Just normalize and return it. Resolve path will
// only be absolute when the path is provided from the command line.
if (cwk_path_is_absolute(path)) { if (cwk_path_is_absolute(path)) {
// buff1 = normalized path. +1 for null terminator. // buff1 = normalized path. +1 for null terminator.
size_t size = cwk_path_normalize(path, buff1, sizeof(buff1)) + 1; size_t size = cwk_path_normalize(path, buff1, sizeof(buff1)) + 1;
pathFixWindowsSeperator(buff1);
char* ret = pkAllocString(vm, size); return tryImportPaths(vm, buff1, buff2);
memcpy(ret, buff1, size);
return ret;
} }
if (from == NULL) { //< [path] is relative to cwd. if (from == NULL) { //< [path] is relative to cwd.
@ -69,10 +146,9 @@ char* pathResolveImport(PKVM* vm, const char* from, const char* path) {
// buff2 = normalized path. +1 for null terminator. // buff2 = normalized path. +1 for null terminator.
size_t size = cwk_path_normalize(buff1, buff2, sizeof(buff2)) + 1; size_t size = cwk_path_normalize(buff1, buff2, sizeof(buff2)) + 1;
pathFixWindowsSeperator(buff2);
char* ret = pkAllocString(vm, size); return tryImportPaths(vm, buff2, buff1);
memcpy(ret, buff2, size);
return ret;
} }
// Import statements doesn't support absolute paths. // Import statements doesn't support absolute paths.
@ -93,12 +169,9 @@ char* pathResolveImport(PKVM* vm, const char* from, const char* path) {
// buff1 = normalized absolute path. +1 for null terminator // buff1 = normalized absolute path. +1 for null terminator
size_t size = cwk_path_normalize(buff2, buff1, sizeof(buff1)) + 1; size_t size = cwk_path_normalize(buff2, buff1, sizeof(buff1)) + 1;
pathFixWindowsSeperator(buff1);
if (pathIsFileExists(buff1)) { return tryImportPaths(vm, buff1, buff2);
char* ret = pkAllocString(vm, size);
memcpy(ret, buff1, size);
return ret;
}
} }
// Cannot resolve the path. Return NULL to indicate failure. // Cannot resolve the path. Return NULL to indicate failure.
@ -109,38 +182,27 @@ char* pathResolveImport(PKVM* vm, const char* from, const char* path) {
/* PATH INTERNAL FUNCTIONS */ /* PATH INTERNAL FUNCTIONS */
/*****************************************************************************/ /*****************************************************************************/
// FIXME: refactor (ref: https://stackoverflow.com/a/230068/10846399) static inline bool pathIsFile(const char* path) {
static inline bool pathIsFileExists(const char* path) { struct stat path_stat;
FILE* file = fopen(path, "r"); stat(path, &path_stat);
if (file != NULL) { return (path_stat.st_mode & S_IFMT) == S_IFREG;
fclose(file);
return true;
}
return false;
} }
// Reference: https://stackoverflow.com/a/12510903/10846399 static inline bool pathIsDir(const char* path) {
static inline bool pathIsDirectoryExists(const char* path) { struct stat path_stat;
DIR* dir = opendir(path); stat(path, &path_stat);
if (dir) { /* Directory exists. */ return (path_stat.st_mode & S_IFMT) == S_IFDIR;
closedir(dir);
return true;
} else if (errno == ENOENT) { /* Directory does not exist. */
} else { /* opendir() failed for some other reason. */
}
return false;
} }
static inline bool pathIsExists(const char* path) { static inline bool pathIsExists(const char* path) {
return pathIsFileExists(path) || pathIsDirectoryExists(path); return access(path, F_OK) == 0;
} }
static inline size_t pathAbs(const char* path, char* buff, size_t buff_size) { static inline size_t pathAbs(const char* path, char* buff, size_t buff_size) {
char cwd[MAX_PATH_LEN]; char cwd[MAX_PATH_LEN];
if (get_cwd(cwd, sizeof(cwd)) == NULL) { if (getcwd(cwd, sizeof(cwd)) == NULL) {
// TODO: handle error. // TODO: handle error.
} }
@ -151,15 +213,9 @@ static inline size_t pathAbs(const char* path, char* buff, size_t buff_size) {
/* PATH MODULE FUNCTIONS */ /* PATH MODULE FUNCTIONS */
/*****************************************************************************/ /*****************************************************************************/
DEF(_pathSetStyleUnix, "") {
bool value;
if (!pkValidateSlotBool(vm, 1, &value)) return;
cwk_path_set_style((value) ? CWK_STYLE_UNIX : CWK_STYLE_WINDOWS);
}
DEF(_pathGetCWD, "") { DEF(_pathGetCWD, "") {
char cwd[MAX_PATH_LEN]; char cwd[MAX_PATH_LEN];
if (get_cwd(cwd, sizeof(cwd)) == NULL) { if (getcwd(cwd, sizeof(cwd)) == NULL) {
// TODO: Handle error. // TODO: Handle error.
} }
pkSetSlotString(vm, 0, cwd); pkSetSlotString(vm, 0, cwd);
@ -269,19 +325,18 @@ DEF(_pathExists, "") {
DEF(_pathIsFile, "") { DEF(_pathIsFile, "") {
const char* path; const char* path;
if (!pkValidateSlotString(vm, 1, &path, NULL)) return; if (!pkValidateSlotString(vm, 1, &path, NULL)) return;
pkSetSlotBool(vm, 0, pathIsFileExists(path)); pkSetSlotBool(vm, 0, pathIsFile(path));
} }
DEF(_pathIsDir, "") { DEF(_pathIsDir, "") {
const char* path; const char* path;
if (!pkValidateSlotString(vm, 1, &path, NULL)) return; if (!pkValidateSlotString(vm, 1, &path, NULL)) return;
pkSetSlotBool(vm, 0, pathIsDirectoryExists(path)); pkSetSlotBool(vm, 0, pathIsDir(path));
} }
void registerModulePath(PKVM* vm) { void registerModulePath(PKVM* vm) {
PkHandle* path = pkNewModule(vm, "path"); PkHandle* path = pkNewModule(vm, "path");
pkModuleAddFunction(vm, path, "setunix", _pathSetStyleUnix, 1);
pkModuleAddFunction(vm, path, "getcwd", _pathGetCWD, 0); pkModuleAddFunction(vm, path, "getcwd", _pathGetCWD, 0);
pkModuleAddFunction(vm, path, "abspath", _pathAbspath, 1); pkModuleAddFunction(vm, path, "abspath", _pathAbspath, 1);
pkModuleAddFunction(vm, path, "relpath", _pathRelpath, 2); pkModuleAddFunction(vm, path, "relpath", _pathRelpath, 2);

467
src/pk_compiler.c
View File

@ -108,7 +108,6 @@ typedef enum {
TK_SLEFTEQ, // <<= TK_SLEFTEQ, // <<=
// Keywords. // Keywords.
TK_MODULE, // module
TK_CLASS, // class TK_CLASS, // class
TK_FROM, // from TK_FROM, // from
TK_IMPORT, // import TK_IMPORT, // import
@ -173,7 +172,6 @@ typedef struct {
// List of keywords mapped into their identifiers. // List of keywords mapped into their identifiers.
static _Keyword _keywords[] = { static _Keyword _keywords[] = {
{ "module", 6, TK_MODULE },
{ "class", 5, TK_CLASS }, { "class", 5, TK_CLASS },
{ "from", 4, TK_FROM }, { "from", 4, TK_FROM },
{ "import", 6, TK_IMPORT }, { "import", 6, TK_IMPORT },
@ -1585,7 +1583,6 @@ GrammarRule rules[] = { // Prefix Infix Infix Precedence
/* TK_SLEFT */ { NULL, exprBinaryOp, PREC_BITWISE_SHIFT }, /* TK_SLEFT */ { NULL, exprBinaryOp, PREC_BITWISE_SHIFT },
/* TK_SRIGHTEQ */ NO_RULE, /* TK_SRIGHTEQ */ NO_RULE,
/* TK_SLEFTEQ */ NO_RULE, /* TK_SLEFTEQ */ NO_RULE,
/* TK_MODULE */ NO_RULE,
/* TK_CLASS */ NO_RULE, /* TK_CLASS */ NO_RULE,
/* TK_FROM */ NO_RULE, /* TK_FROM */ NO_RULE,
/* TK_IMPORT */ NO_RULE, /* TK_IMPORT */ NO_RULE,
@ -2792,349 +2789,146 @@ static void compileBlockBody(Compiler* compiler, BlockType type) {
compilerExitBlock(compiler); compilerExitBlock(compiler);
} }
// Import a file at the given path (first it'll be resolved from the current // Parse the module path syntax, emit opcode to load module at that path.
// path) and return it as a module pointer. And it'll emit opcodes to push // and return the module's name token
// that module to the stack. //
static Module* importFile(Compiler* compiler, const char* path) { // ex: import foo.bar.baz // => "foo/bar/baz" => return token 'baz'
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS); // import .qux.lex // => "./qux/lex" => return token 'lex'
// import ^^foo.bar // => "../../foo/bar" => return token 'bar'
//
// The name start pointer and its length will be written to the parameters.
// For invalid syntax it'll set an error and return an error token.
static Token compileImportPath(Compiler* compiler) {
PKVM* vm = compiler->parser.vm; PKVM* vm = compiler->parser.vm;
pkByteBuffer buff; // A buffer to write the path string.
pkByteBufferInit(&buff);
// FIXME: REPL mode mdule doesn't have a path, So we're using NULL and the if (match(compiler, TK_DOT)) {
// resolve path function will use cwd to resolve. pkByteBufferAddString(&buff, vm, "./", 2);
const char* from = NULL;
if (compiler->module->path != NULL) from = compiler->module->path->data;
char* resolved = NULL;
if (vm->config.resolve_path_fn != NULL) {
resolved = vm->config.resolve_path_fn(vm, from, path);
}
if (resolved == NULL) {
semanticError(compiler, compiler->parser.previous,
"Cannot resolve path '%s' from '%s'", path, from);
return NULL;
}
// Create new string for the resolved path. And free the resolved path.
int index = 0;
String* path_ = moduleAddString(compiler->module, compiler->parser.vm,
resolved,
(uint32_t)strlen(resolved),
&index);
pkDeAllocString(vm, resolved);
// Check if the script already compiled and cached in the PKVM.
Var entry = mapGet(vm->modules, VAR_OBJ(path_));
if (!IS_UNDEF(entry)) {
ASSERT(IS_OBJ_TYPE(entry, OBJ_MODULE), OOPS);
// Push the compiled script on the stack.
emitOpcode(compiler, OP_IMPORT);
emitShort(compiler, index);
return (Module*)AS_OBJ(entry);
}
// The script not exists in the VM, make sure we have the script loading
// api function.
if (vm->config.load_script_fn == NULL) {
semanticError(compiler, compiler->parser.previous,
"Cannot import. The hosting application haven't registered "
"the script loading API");
return NULL;
}
// Load the script at the path. Note that if the source is not NULL, it's
// our responsibility to deallocate the string with the pkDeallocString()
// function.
char* source = vm->config.load_script_fn(vm, path_->data);
if (source == NULL) {
semanticError(compiler, compiler->parser.previous,
"Error loading script at \"%s\"", path_->data);
return NULL;
}
// Make a new module and to compile it.
Module* module = newModule(vm);
module->path = path_;
vmPushTempRef(vm, &module->_super); // module.
initializeScript(vm, module);
vmRegisterModule(vm, module, path_);
vmPopTempRef(vm); // module.
// Push the compiled script on the stack.
emitOpcode(compiler, OP_IMPORT);
emitShort(compiler, index);
// Even if we're running on repl mode the imported module cannot run on
// repl mode.
CompileOptions options = newCompilerOptions();
if (compiler->options) options = *compiler->options;
options.repl_mode = false;
// Compile the source to the module and clean the source.
PkResult result = compile(vm, module, source, &options);
pkDeAllocString(vm, source);
if (result != PK_RESULT_SUCCESS) {
semanticError(compiler, compiler->parser.previous,
"Compilation of imported script '%s' failed", path_->data);
}
return module;
}
// Import the native module from the PKVM's core_libs and it'll emit opcodes
// to push that module to the stack.
static Module* importCoreLib(Compiler* compiler, const char* name_start,
int name_length) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
// Add the name to the module's name buffer, we need it as a key to the
// PKVM's module cache.
int index = 0;
String* module_name = moduleAddString(compiler->module, compiler->parser.vm,
name_start, name_length, &index);
Module* imported = vmGetModule(compiler->parser.vm, module_name);
if (imported == NULL) {
semanticError(compiler, compiler->parser.previous,
"No module named '%s' exists.", module_name->data);
return NULL;
}
// Push the module on the stack.
emitOpcode(compiler, OP_IMPORT);
emitShort(compiler, index);
return imported;
}
// Push the imported module on the stack and return the pointer. It could be
// either core library or a local import.
static inline Module* compilerImport(Compiler* compiler) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
// Get the module (from native libs or VM's cache or compile new one).
// And push it on the stack.
if (match(compiler, TK_NAME)) { //< Core library.
return importCoreLib(compiler, compiler->parser.previous.start,
compiler->parser.previous.length);
} else if (match(compiler, TK_STRING)) { //< Local library.
Var var_path = compiler->parser.previous.value;
ASSERT(IS_OBJ_TYPE(var_path, OBJ_STRING), OOPS);
String* path = (String*)AS_OBJ(var_path);
return importFile(compiler, path->data);
}
// Invalid token after import/from keyword.
syntaxError(compiler, compiler->parser.previous,
"Expected a module name or path to import.");
return NULL;
}
// Search for the name, and return it's index in the globals. If it's not
// exists in the globals it'll add a variable to the globals entry and return.
// But If the name is predefined function (cannot be modified). It'll set error
// and return -1.
static int compilerImportName(Compiler* compiler, int line,
const char* name, uint32_t length) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
NameSearchResult result = compilerSearchName(compiler, name, length);
switch (result.type) {
case NAME_NOT_DEFINED:
return compilerAddVariable(compiler, name, length, line);
case NAME_LOCAL_VAR:
case NAME_UPVALUE:
UNREACHABLE();
case NAME_GLOBAL_VAR:
return result.index;
// TODO:
// Make it possible to override any name (ie. the syntax `print = 1`
// should pass) and allow imported entries to have the same name of
// builtin functions.
case NAME_BUILTIN_FN:
case NAME_BUILTIN_TY:
semanticError(compiler, compiler->parser.previous,
"Name '%.*s' already exists.", length, name);
return -1;
}
UNREACHABLE();
return -1;
}
// This will called by the compilerImportAll() function to import a single
// entry from the imported module. (could be a function or global variable).
static void compilerImportSingleEntry(Compiler* compiler,
const char* name, uint32_t length) {
// Special names are begins with '@' (implicit main function, literal
// functions etc) skip them.
if (name[0] == SPECIAL_NAME_CHAR) return;
// Line number of the variables which will be bind to the imported symbol.
int line = compiler->parser.previous.line;
// Add the name to the **current** module's name buffer.
int name_index = 0;
moduleAddString(compiler->module, compiler->parser.vm,
name, length, &name_index);
// Get the global/function/class from the module.
emitOpcode(compiler, OP_GET_ATTRIB_KEEP);
emitShort(compiler, name_index);
int index = compilerImportName(compiler, line, name, length);
if (index != -1) emitStoreGlobal(compiler, index);
emitOpcode(compiler, OP_POP);
}
// Import all from the module, which is also would be at the top of the stack
// before executing the below instructions.
static void compilerImportAll(Compiler* compiler, Module* module) {
ASSERT(module != NULL, OOPS);
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
// Import all globals.
ASSERT(module->global_names.count == module->globals.count, OOPS);
for (uint32_t i = 0; i < module->globals.count; i++) {
String* name = moduleGetStringAt(module, module->global_names.data[i]);
ASSERT(name != NULL, OOPS);
// If a name starts with '_' we treat it as private and not importing.
if (name->length >= 1 && name->data[0] == '_') continue;
compilerImportSingleEntry(compiler, name->data, name->length);
}
}
// from module import symbol [as alias [, symbol2 [as alias]]]
static void compileFromImport(Compiler* compiler) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
// Import the library and push it on the stack. If the import failed
// lib_from would be NULL.
Module* lib_from = compilerImport(compiler);
// At this point the module would be on the stack before executing the next
// instruction.
consume(compiler, TK_IMPORT, "Expected keyword 'import'.");
if (match(compiler, TK_STAR)) {
// from math import *
if (lib_from) compilerImportAll(compiler, lib_from);
} else { } else {
do { // Consume parent directory syntax.
// Consume the symbol name to import from the module. while (match(compiler, TK_CARET)) {
consume(compiler, TK_NAME, "Expected symbol to import."); pkByteBufferAddString(&buff, vm, "../", 3);
const char* name = compiler->parser.previous.start; }
uint32_t length = (uint32_t)compiler->parser.previous.length;
int line = compiler->parser.previous.line;
// Add the name of the symbol to the names buffer.
int name_index = 0;
moduleAddString(compiler->module, compiler->parser.vm,
name, length, &name_index);
// Don't pop the lib since it'll be used for the next entry.
emitOpcode(compiler, OP_GET_ATTRIB_KEEP);
emitShort(compiler, name_index); //< Name of the attrib.
// Check if it has an alias.
if (match(compiler, TK_AS)) {
// Consuming it'll update the previous token which would be the name of
// the binding variable.
consume(compiler, TK_NAME, "Expected a name after 'as'.");
}
// Set the imported symbol binding name, which wold be in the last token
// consumed by the first one or after the as keyword.
name = compiler->parser.previous.start;
length = (uint32_t)compiler->parser.previous.length;
line = compiler->parser.previous.line;
// Get the variable to bind the imported symbol, if we already have a
// variable with that name override it, otherwise use a new variable.
int var_index = compilerImportName(compiler, line, name, length);
if (var_index != -1) emitStoreGlobal(compiler, var_index);
emitOpcode(compiler, OP_POP);
} while (match(compiler, TK_COMMA) && (skipNewLines(compiler), true));
} }
// Done getting all the attributes, now pop the lib from the stack. Token tkmodule = makeErrToken(&compiler->parser);
emitOpcode(compiler, OP_POP);
// Consume module path.
do {
consume(compiler, TK_NAME, "Expected a module name");
if (compiler->parser.has_syntax_error) break;
// A '.' consumed, write '/'.
if (tkmodule.type != TK_ERROR) pkByteBufferWrite(&buff, vm, (uint8_t) '/');
tkmodule = compiler->parser.previous;
pkByteBufferAddString(&buff, vm, tkmodule.start, tkmodule.length);
} while (match(compiler, TK_DOT));
pkByteBufferWrite(&buff, vm, '\0');
if (compiler->parser.has_syntax_error) {
pkByteBufferClear(&buff, vm);
return makeErrToken(&compiler->parser);
}
// Create constant pool entry for the path string.
int index = 0;
moduleAddString(compiler->module, compiler->parser.vm,
buff.data, buff.count - 1, &index);
pkByteBufferClear(&buff, vm);
emitOpcode(compiler, OP_IMPORT);
emitShort(compiler, index);
return tkmodule;
}
// import module1 [as alias1 [, module2 [as alias2 ...]]
void compileRegularImport(Compiler* compiler) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
do {
Token tkmodule = compileImportPath(compiler);
if (tkmodule.type == TK_ERROR) return; //< Syntax error. Terminate.
if (match(compiler, TK_AS)) {
consume(compiler, TK_NAME, "Expected a name after 'as'.");
if (compiler->parser.has_syntax_error) return;
tkmodule = compiler->parser.previous;
}
// FIXME:
// Note that for compilerAddVariable for adding global doesn't create
// a new global variable if it's already exists. But it'll reuse it. So we
// don't have to check if it's exists (unlike locals) which is an
// inconsistance behavior IMO. The problem here is that compilerAddVariable
// will try to initialize the global with VAR_NULL which may not be
// accceptable in some scenarios,
int global_index = compilerAddVariable(compiler, tkmodule.start,
tkmodule.length, tkmodule.line);
emitStoreGlobal(compiler, global_index);
emitOpcode(compiler, OP_POP);
} while (match(compiler, TK_COMMA) && (skipNewLines(compiler), true));
// Always end the import statement. // Always end the import statement.
consumeEndStatement(compiler); consumeEndStatement(compiler);
} }
static void compileRegularImport(Compiler* compiler) { // from module import sym1 [as alias1 [, sym2 [as alias2 ...]]]
static void compileFromImport(Compiler* compiler) {
ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS); ASSERT(compiler->scope_depth == DEPTH_GLOBAL, OOPS);
Token tkmodule = compileImportPath(compiler);
if (tkmodule.type == TK_ERROR) return; //< Syntax error. Terminate.
// At this point the module would be on the stack before executing the next
// instruction.
consume(compiler, TK_IMPORT, "Expected keyword 'import'.");
if (compiler->parser.has_syntax_error) return;
do { do {
// Consume the symbol name to import from the module.
consume(compiler, TK_NAME, "Expected symbol to import.");
if (compiler->parser.has_syntax_error) return;
Token tkname = compiler->parser.previous;
// Import the library and push it on the stack. If it cannot import, // Add the name of the symbol to the constant pool.
// the lib would be null, but we're not terminating here, just continue int name_index = 0;
// parsing for cascaded errors. moduleAddString(compiler->module, compiler->parser.vm, tkname.start,
Module* lib = compilerImport(compiler); tkname.length, &name_index);
// variable to bind the imported module. // Don't pop the lib since it'll be used for the next entry.
int var_index = -1; emitOpcode(compiler, OP_GET_ATTRIB_KEEP);
emitShort(compiler, name_index); //< Name of the attrib.
// Check if it has an alias, if so bind the variable with that name. // Check if it has an alias.
if (match(compiler, TK_AS)) { if (match(compiler, TK_AS)) {
// Consuming it'll update the previous token which would be the name of // Consuming it'll update the previous token which would be the name of
// the binding variable. // the binding variable.
consume(compiler, TK_NAME, "Expected a name after 'as'."); consume(compiler, TK_NAME, "Expected a name after 'as'.");
tkname = compiler->parser.previous;
// Get the variable to bind the imported symbol, if we already have a
// variable with that name override it, otherwise use a new variable.
const char* name = compiler->parser.previous.start;
int length = compiler->parser.previous.length;
int line = compiler->parser.previous.line;
var_index = compilerImportName(compiler, line, name, length);
} else {
// If it has a module name use it as binding variable.
// Core libs names are it's module name but for local libs it's optional
// to define a module name for a module.
if (lib && lib->name != NULL) {
// Get the variable to bind the imported symbol, if we already have a
// variable with that name override it, otherwise use a new variable.
const char* name = lib->name->data;
uint32_t length = lib->name->length;
int line = compiler->parser.previous.line;
var_index = compilerImportName(compiler, line, name, length);
} else {
// -- Nothing to do here --
// Importing from path which doesn't have a module name. Import
// everything of it. and bind to a variables.
NO_OP;
}
} }
if (var_index != -1) { // FIXME: See the same FIXME for compilerAddVariable()
emitStoreGlobal(compiler, var_index); // compileRegularImport function.
emitOpcode(compiler, OP_POP); int global_index = compilerAddVariable(compiler, tkname.start,
tkname.length, tkname.line);
} else { emitStoreGlobal(compiler, global_index);
if (lib) compilerImportAll(compiler, lib); emitOpcode(compiler, OP_POP);
// Done importing everything from lib now pop the lib.
emitOpcode(compiler, OP_POP);
}
} while (match(compiler, TK_COMMA) && (skipNewLines(compiler), true)); } while (match(compiler, TK_COMMA) && (skipNewLines(compiler), true));
// Done getting all the attributes, now pop the lib from the stack.
emitOpcode(compiler, OP_POP);
// Always end the import statement.
consumeEndStatement(compiler); consumeEndStatement(compiler);
} }
@ -3404,15 +3198,11 @@ static void compileTopLevelStatement(Compiler* compiler) {
} else if (match(compiler, TK_DEF)) { } else if (match(compiler, TK_DEF)) {
compileFunction(compiler, FUNC_TOPLEVEL); compileFunction(compiler, FUNC_TOPLEVEL);
} else if (match(compiler, TK_FROM)) {
compileFromImport(compiler);
} else if (match(compiler, TK_IMPORT)) { } else if (match(compiler, TK_IMPORT)) {
compileRegularImport(compiler); compileRegularImport(compiler);
} else if (match(compiler, TK_MODULE)) { } else if (match(compiler, TK_FROM)) {
syntaxError(compiler, compiler->parser.previous, compileFromImport(compiler);
"Module name must be the first statement of the script.");
} else { } else {
compileStatement(compiler); compileStatement(compiler);
@ -3473,24 +3263,6 @@ PkResult compile(PKVM* vm, Module* module, const char* source,
lexToken(compiler); lexToken(compiler);
skipNewLines(compiler); skipNewLines(compiler);
if (match(compiler, TK_MODULE)) {
// If the module running a REPL or compiled multiple times by hosting
// application module attribute might already set. In that case make it
// Compile error.
if (module->name != NULL) {
syntaxError(compiler, compiler->parser.previous,
"Module name already defined.");
} else {
consume(compiler, TK_NAME, "Expected a name for the module.");
const char* name = compiler->parser.previous.start;
uint32_t len = compiler->parser.previous.length;
module->name = newStringLength(vm, name, len);
consumeEndStatement(compiler);
}
}
while (!match(compiler, TK_EOF) && !compiler->parser.has_syntax_error) { while (!match(compiler, TK_EOF) && !compiler->parser.has_syntax_error) {
compileTopLevelStatement(compiler); compileTopLevelStatement(compiler);
skipNewLines(compiler); skipNewLines(compiler);
@ -3526,10 +3298,13 @@ PkResult compile(PKVM* vm, Module* module, const char* source,
module->constants.count = constants_count; module->constants.count = constants_count;
module->globals.count = module->global_names.count = globals_count; module->globals.count = module->global_names.count = globals_count;
} }
#if DUMP_BYTECODE #if DUMP_BYTECODE
dumpFunctionCode(compiler->parser.vm, module->body->fn); else {
DEBUG_BREAK(); // If there is any syntax errors we cannot dump the bytecode
// (otherwise it'll crash with assertion).
dumpFunctionCode(compiler->parser.vm, module->body->fn);
DEBUG_BREAK();
}
#endif #endif
// Return the compilation result. // Return the compilation result.

3
src/pk_public.c
View File

@ -285,9 +285,6 @@ PkResult pkRunString(PKVM* vm, const char* source) {
PkResult pkRunFile(PKVM* vm, const char* path) { PkResult pkRunFile(PKVM* vm, const char* path) {
// FIXME(_imp_): path resolve function should always expect a source file
// path (ends with .pk) to be consistance with the paths.
// Note: The file may have been imported by some other script and cached in // Note: The file may have been imported by some other script and cached in
// the VM's scripts cache. But we're not using that instead, we'll recompile // the VM's scripts cache. But we're not using that instead, we'll recompile
// the file and update the cache. // the file and update the cache.

120
src/pk_vm.c
View File

@ -334,31 +334,102 @@ PkResult vmRunFunction(PKVM* vm, Closure* fn, int argc, Var* argv, Var* ret) {
/* VM INTERNALS */ /* VM INTERNALS */
/*****************************************************************************/ /*****************************************************************************/
// FIXME: // Import and return the Module object with the [path] string. If the path
// We're assuming that the module should be available at the VM's modules cache // starts with with './' or '../' we'll only try relative imports, otherwise
// which is added by the compilation pahse, but we cannot rely on the // we'll search native modules first and then at relative path.
// compilation phase here as it could be a seperate system from the runtime and static inline Var importModule(PKVM* vm, String* from, String* path) {
// should throw a runtime error if the module is not present in the modules ASSERT((path != NULL) && (path->length > 0), OOPS);
// cache (or try to load).
// Example: If we may support to store the compiled script as a separate file
// (like python's ".pyc" or java's ".class" the runtime cannot ensure that the
// module it import is already cached.
//
// Import and return the Module object with the [name] (if it's a scirpt
// doesn't have a module name, the name would be it's resolved path).
static inline Var importModule(PKVM* vm, String* key) {
Var entry = mapGet(vm->modules, VAR_OBJ(key)); bool is_relative = path->data[0] == '.';
if (!IS_UNDEF(entry)) {
ASSERT(AS_OBJ(entry)->type == OBJ_MODULE, OOPS); // If not relative check the [path] in the modules cache with the name
return entry; // (before resolving the path).
if (!is_relative) {
// If not relative path we first search in modules cache. It'll find the
// native module or the already imported cache of the script.
Var entry = mapGet(vm->modules, VAR_OBJ(path));
if (!IS_UNDEF(entry)) {
ASSERT(AS_OBJ(entry)->type == OBJ_MODULE, OOPS);
return entry; // We're done.
}
} }
// FIXME: Should be a runtime error. // If we reached here. It's not a native module (ie. module's absolute path
// Imported modules were resolved at compile time. // is required to import and cache).
UNREACHABLE(); char* _resolved = vm->config.resolve_path_fn(vm,
(from) ? from->data : NULL, path->data);
return VAR_NULL; if (_resolved == NULL) { // Can't resolve a relative module.
pkDeAllocString(vm, _resolved);
if (from) {
VM_SET_ERROR(vm, stringFormat(vm, "Cannot resolve a relative import "
"path \"@\" from \"@\".", path, from));
} else {
VM_SET_ERROR(vm, stringFormat(vm, "Cannot resolve a relative import "
"path \"@\"", path));
}
return VAR_NULL;
}
String* resolved = newString(vm, _resolved);
pkDeAllocString(vm, _resolved);
// If the script already imported and cached, return it.
Var entry = mapGet(vm->modules, VAR_OBJ(resolved));
if (!IS_UNDEF(entry)) {
ASSERT(AS_OBJ(entry)->type == OBJ_MODULE, OOPS);
return entry; // We're done.
}
// If we've reached here, the script is new. Import compile, and cache it.
// The script not exists in the VM, make sure we have the script loading
// api function.
if (vm->config.load_script_fn == NULL) {
VM_SET_ERROR(vm, newString(vm, "Cannot import. The hosting application "
"haven't registered the module loading API"));
return VAR_NULL;
}
Module* module = NULL;
vmPushTempRef(vm, &resolved->_super); // resolved.
do {
char* source = vm->config.load_script_fn(vm, resolved->data);
if (source == NULL) {
VM_SET_ERROR(vm, stringFormat(vm, "Error loading module at \"@\"",
resolved));
break;
}
// Make a new module and to compile it.
module = newModule(vm);
module->path = resolved;
vmPushTempRef(vm, &module->_super); // module.
{
initializeScript(vm, module);
PkResult result = compile(vm, module, source, NULL);
pkDeAllocString(vm, source);
if (result == PK_RESULT_SUCCESS) {
vmRegisterModule(vm, module, resolved);
} else {
VM_SET_ERROR(vm, stringFormat(vm, "Error compiling module at \"@\"",
resolved));
module = NULL; //< set to null to indicate error.
}
}
vmPopTempRef(vm); // module.
} while (false);
vmPopTempRef(vm); // resolved.
if (module == NULL) {
ASSERT(VM_HAS_ERROR(vm), OOPS);
return VAR_NULL;
}
return VAR_OBJ(module);
} }
void vmEnsureStackSize(PKVM* vm, int size) { void vmEnsureStackSize(PKVM* vm, int size) {
@ -952,12 +1023,11 @@ L_vm_main_loop:
String* name = moduleGetStringAt(module, (int)index); String* name = moduleGetStringAt(module, (int)index);
ASSERT(name != NULL, OOPS); ASSERT(name != NULL, OOPS);
Var _imported = importModule(vm, name); Var _imported = importModule(vm, module->path, name);
CHECK_ERROR();
ASSERT(IS_OBJ_TYPE(_imported, OBJ_MODULE), OOPS); ASSERT(IS_OBJ_TYPE(_imported, OBJ_MODULE), OOPS);
PUSH(_imported);
// TODO: If the body doesn't have any statements (just the functions). PUSH(_imported);
// This initialization call is un-necessary.
Module* imported = (Module*)AS_OBJ(_imported); Module* imported = (Module*)AS_OBJ(_imported);
if (!imported->initialized) { if (!imported->initialized) {

48
tests/lang/import.pk
View File

@ -6,40 +6,32 @@ import lang as o, path as p
from lang import write from lang import write
from lang import clock as c from lang import clock as c
from lang import * import basics
from path import * import controlflow as if_test
from functions import f1, f2 as fn2, f3
import "basics.pk" ## will import all import imports.fns
import "controlflow.pk" as if_test assert(fns.f1() == 'f1')
from "functions.pk" import f1, f2 as fn2, f3 assert(fns.f2() == 'f2')
assert(fns.f3() == 'f3')
## If it has a module name it'll bind to that name.
import 'import/module.pk'
assert(module_name.get_module_name() == 'module_name')
## Import everything from the module.
from 'import/module.pk' import *
assert(module_name.get_module_name == get_module_name)
## script without module name will import all by default.
import 'import/all_import.pk'
assert(all_f1() == 'f1')
assert(all_f2() == 'f2')
assert(all_f3() == 'f3')
## Import the script and bound it with a given name. ## Import the script and bound it with a given name.
import 'import/all_import.pk' as all_import import imports.fns as fns2
assert(all_import.all_f1 == all_f1) assert(fns2 == fns)
assert(fns2.f1 == fns.f1)
## Test if the imported globals were initialized ## Test if the imported globals were initialized
import 'import/globals.pk' import imports.globals as gbl
assert(g_import != null) assert(gbl.g_1 == 3)
assert(g_import.g_var_1 == 3) assert(gbl.g_2 == gbl.get_a_value())
assert(g_import.g_var_2 == g_import.get_a_value())
import 'import/globals2.pk' ## import "imports/foo/_init.pk::bar"
assert(g_val_1 == 100) from imports.foo import bar
assert(g_val_2 == get_a_value()) assert(bar() == "foo.bar")
## Test cyclic import
import imports.cyclic_a as a
assert(a.get_b_fn()() == "cyclic b")
# If we got here, that means all test were passed. # If we got here, that means all test were passed.
print('All TESTS PASSED') print('All TESTS PASSED')

15
tests/lang/import/all_import.pk
View File

@ -1,15 +0,0 @@
## TODO: initialize global variables
## and test it here.
def all_f1()
return 'f1'
end
def all_f2()
return 'f2'
end
def all_f3()
return 'f3'
end

10
tests/lang/import/globals.pk
View File

@ -1,10 +0,0 @@
module g_import
def get_a_value()
return "foobar"
end
g_var_1 = 1 + 2
g_var_2 = get_a_value()

12
tests/lang/import/globals2.pk
View File

@ -1,12 +0,0 @@
def get_a_value()
return "baz"
end
g_val_1 = 0
for i in 0..100
g_val_1 += 1
end
g_val_2 = get_a_value()

6
tests/lang/import/module.pk
View File

@ -1,6 +0,0 @@
module module_name
def get_module_name()
return 'module_name'
end

12
tests/lang/imports/cyclic_a.pk Normal file
View File

@ -0,0 +1,12 @@
print('before cyclic_a')
import cyclic_b
print('after cyclic_a')
def fn_a()
return "cyclic a"
end
def get_b_fn()
return cyclic_b.fn_b
end

10
tests/lang/imports/cyclic_b.pk Normal file
View File

@ -0,0 +1,10 @@
print('before cyclic_b')
import cyclic_a
print('after cyclic_b')
def fn_b()
assert(cyclic_a.fn_a() == "cyclic a")
return "cyclic b"
end

12
tests/lang/imports/fns.pk Normal file
View File

@ -0,0 +1,12 @@
def f1()
return 'f1'
end
def f2()
return 'f2'
end
def f3()
return 'f3'
end

9
tests/lang/imports/foo/_init.pk Normal file
View File

@ -0,0 +1,9 @@
## import "../../functions.pk"
import ^^functions as fns
def bar()
assert(fns.f2() == "f2")
return "foo.bar"
end

13
tests/lang/imports/globals.pk Normal file
View File

@ -0,0 +1,13 @@
def one_call()
assert(g_2 == null) ## second import cannot run this.
return get_a_value()
end
def get_a_value()
return "foobar"
end
g_1 = 1 + 2
g_2 = one_call()

7
tests/lang/imports/math.pk Normal file
View File

@ -0,0 +1,7 @@
import math ## core math module
## math script sqrt fn.
def sqrt(n)
return ".math sqrt fn"
end

12
tests/lang/imports/relative.pk Normal file
View File

@ -0,0 +1,12 @@
import .math
assert(math.sqrt(4) == ".math sqrt fn")
import math
assert(math.sqrt(4) == 2)
import ^functions as fns
assert(fns.f1() == "f1")
# If we got here, that means all test were passed.
print('All TESTS PASSED')

36
tests/modules/math.pk
View File

@ -1,5 +1,9 @@
from math import * ## `from math import *` not supported anymore
import math as m
from math import abs, round, log10,
floor, ceil
assert(ceil(1.1) == floor(2.9)) assert(ceil(1.1) == floor(2.9))
@ -8,35 +12,35 @@ assert(ceil(1.1) == floor(2.9))
## and it'll be fixed soon. ## and it'll be fixed soon.
PI = 3.14159265358979323846 PI = 3.14159265358979323846
assert(sin(0) == 0) assert(m.sin(0) == 0)
assert(sin(PI/2) == 1) assert(m.sin(PI/2) == 1)
threshold = 0.0000000000001 threshold = 0.0000000000001
assert(abs(cos(PI/3) - 0.5) < threshold ) assert(abs(m.cos(PI/3) - 0.5) < threshold )
assert(abs(tan(PI/4) - 1.0) < threshold ) assert(abs(m.tan(PI/4) - 1.0) < threshold )
for i in 0..1000 for i in 0..1000
assert(abs(sin(i) / cos(i) - tan(i)) < threshold) assert(abs(m.sin(i) / m.cos(i) - m.tan(i)) < threshold)
end end
assert((cosh(.5) - 1.1276259652063807) < threshold) assert((m.cosh(.5) - 1.1276259652063807) < threshold)
assert((tanh(0.5) - 1.127625965206) < threshold) assert((m.tanh(0.5) - 1.127625965206) < threshold)
for i in 0..100 for i in 0..100
assert(abs(sinh(i) / cosh(i) - tanh(i)) < threshold) assert(abs(m.sinh(i) / m.cosh(i) - m.tanh(i)) < threshold)
end end
assert(abs(acos(PI/4) - 0.5) < 0.35) assert(abs(m.acos(PI/4) - 0.5) < 0.35)
assert(abs(atan(PI/4) - 0.5) < 0.2) assert(abs(m.atan(PI/4) - 0.5) < 0.2)
assert((acos(0.5) - 1.1276259652063807) < threshold) assert((m.acos(0.5) - 1.1276259652063807) < threshold)
assert((atan(0.3) - 1.1276259652063807) < threshold) assert((m.atan(0.3) - 1.1276259652063807) < threshold)
x = -1; interval = 1000 x = -1; interval = 1000
for i in 0..interval-1 for i in 0..interval-1
x += 2/interval x += 2/interval
assert(abs(sin(asin(x)) - x) < threshold) assert(abs(m.sin(m.asin(x)) - x) < threshold)
assert(abs(cos(acos(x)) - x) < threshold) assert(abs(m.cos(m.acos(x)) - x) < threshold)
assert(abs(tan(atan(x)) - x) < threshold) assert(abs(m.tan(m.atan(x)) - x) < threshold)
end end
assert(abs(log10(2) - 0.3010299956639812) < threshold) assert(abs(log10(2) - 0.3010299956639812) < threshold)