feat: workspaces (#89)

* wip

* wip

* fix

* fix lint

* change @dcl dependencies version

* update kernel version

* update min cli version and last kernel PR version

* add timeout to export

* fix lint

* endpoint if scene.json fails

* update kernel version, add TODO comment and change .filter to a single reduce call

* fix typo

* ordering preview wearables

* fix typo

* update kernel

* remove cli version requirement

* add cli version requirement 3.7.0

Author: leanmendoza

packages/decentraland-ecs/package.json

@@ -29,7 +29,7 @@
   "dependencies": {
     "@dcl/amd": "file:../@dcl/amd",
     "@dcl/build-ecs": "file:../@dcl/build-ecs",
-    "@dcl/kernel": "1.0.0-1692945184.commit-4d9fcfd",
+    "@dcl/kernel": "1.0.0-1772786810.commit-feaf312",
     "@dcl/posix": "^1.0.4",
     "@dcl/schemas": "3.5.0",
     "@dcl/unity-renderer": "^1.0.26165-20220128194341.commit-8eaafdc",

packages/decentraland-ecs/src/cli/mock-catalyst/index.ts

@@ -6,10 +6,9 @@ import { getAllPreviewWearables } from '../wearables'
 
 export const mockCatalyst = (
   app: express.Application,
-  baseFolders: string[],
-  rootFolder: string
+  baseFolders: string[]
 ) => {
-  serveFolders(app, baseFolders, rootFolder)
+  serveFolders(app, baseFolders)
   app.get('/lambdas/explore/realms', (req, res) => {
     res.json([
       {
@@ -37,7 +36,6 @@ export const mockCatalyst = (
     try {
       const previewWearables = await getAllPreviewWearables({
         baseFolders,
-        catalystRootFolder: rootFolder,
         baseUrl: ''
       }).map((wearable) => wearable.id)
 
@@ -95,11 +93,7 @@ export const mockCatalyst = (
   )
 }
 
-const serveFolders = (
-  app: express.Application,
-  baseFolders: string[],
-  catalystRootFolder: string
-) => {
+const serveFolders = (app: express.Application, baseFolders: string[]) => {
   app.get('/content/contents/:hash', (req, res, next) => {
     if (req.params.hash && req.params.hash.startsWith('b64-')) {
       const fullPath = path.resolve(
@@ -110,7 +104,7 @@ const serveFolders = (
 
       // only return files IF the file is within a baseFolder
       if (!baseFolders.find((folder: string) => fullPath.startsWith(folder))) {
-        res.end(404)
+        next()
         return
       }
 
@@ -149,8 +143,7 @@ const serveFolders = (
 
     const resultEntities = getSceneJson({
       baseFolders,
-      pointers: Array.from(requestedPointers),
-      catalystRootFolder
+      pointers: Array.from(requestedPointers)
     })
     res.json(resultEntities).end()
   })

packages/decentraland-ecs/src/cli/setupUtils.ts

@@ -30,14 +30,12 @@ export const getFilesFromFolder = ({
   folder,
   addOriginalPath,
   ignorePattern,
-  customHashMaker,
-  rootFolder
+  customHashMaker
 }: {
   folder: string
   addOriginalPath?: boolean
   ignorePattern?: string
   customHashMaker?: (str: string) => string
-  rootFolder?: string
 }) => {
   const hashMaker = customHashMaker ? customHashMaker : defaultHashMaker
 
@@ -73,20 +71,16 @@ export const getFilesFromFolder = ({
         return
       }
 
-      const absoluteFolder = rootFolder
-        ? rootFolder.replace(/\\/gi, '/')
-        : folder.replace(/\\/gi, '/')
-      const relativeFilePathToRootFolder = absolutePath
-        .replace(absoluteFolder, '')
-        .replace(/^\/+/, '')
+      const absoluteFolder = folder.replace(/\\/gi, '/')
+
       const relativeFilePathToFolder = file
         .replace(absoluteFolder, '')
         .replace(/^\/+/, '')
 
       return {
         file: relativeFilePathToFolder.toLowerCase(),
         original_path: addOriginalPath ? absolutePath : undefined,
-        hash: hashMaker(relativeFilePathToRootFolder)
+        hash: hashMaker(absolutePath)
       }
     })
     .filter(($) => !!$)
@@ -96,14 +90,12 @@ export function entityV3FromFolder({
   folder,
   addOriginalPath,
   ignorePattern,
-  customHashMaker,
-  catalystRootFolder
+  customHashMaker
 }: {
   folder: string
   addOriginalPath?: boolean
   ignorePattern?: string
   customHashMaker?: (str: string) => string
-  catalystRootFolder: string
 }) {
   const sceneJsonPath = path.resolve(folder, './scene.json')
   let isParcelScene = true
@@ -128,7 +120,6 @@ export function entityV3FromFolder({
 
   if (fs.existsSync(sceneJsonPath) && isParcelScene) {
     const sceneJson = JSON.parse(fs.readFileSync(sceneJsonPath).toString())
-
     const { base, parcels }: { base: string; parcels: string[] } =
       sceneJson.scene
     const pointers = new Set<string>()
@@ -139,8 +130,7 @@ export function entityV3FromFolder({
       folder,
       addOriginalPath,
       ignorePattern,
-      customHashMaker,
-      rootFolder: catalystRootFolder
+      customHashMaker
     })
     return {
       version: 'v3',
@@ -159,13 +149,11 @@ export function entityV3FromFolder({
 export function getSceneJson({
   baseFolders,
   pointers,
-  customHashMaker,
-  catalystRootFolder
+  customHashMaker
 }: {
   baseFolders: string[]
   pointers: string[]
   customHashMaker?: (str: string) => string
-  catalystRootFolder: string
 }) {
   const requestedPointers = new Set<string>(pointers)
   const resultEntities = []
@@ -183,8 +171,7 @@ export function getSceneJson({
       folder,
       addOriginalPath: false,
       ignorePattern: ignoreFileContent,
-      customHashMaker,
-      catalystRootFolder
+      customHashMaker
     })
   })
 
@@ -224,7 +211,11 @@ export async function ensureCopyFile(fromFilePath: string, filePath: any) {
   await fs.promises.copyFile(fromFilePath, filePath)
 }
 
-export const downloadFile = async (url: string, path: string) => {
+export const downloadFile = async (
+  url: string,
+  path: string,
+  timeout_seg: number = 15
+) => {
   return new Promise((resolve, reject) => {
     const file = fs.createWriteStream(path)
 
@@ -233,18 +224,28 @@ export const downloadFile = async (url: string, path: string) => {
       schema = https as any
     }
 
-    schema
+    let finished = false
+    const request = schema
       .get(url, function (response) {
         response.pipe(file)
         file.on('finish', function () {
           file.close()
+          finished = true
           resolve(true)
         })
       })
       .on('error', function (err) {
         fs.unlinkSync(path)
+        finished = true
         reject(err)
       })
+
+    setTimeout(() => {
+      if (!finished) {
+        request.destroy()
+        reject(new Error(`Timeout ${url}`))
+      }
+    }, timeout_seg * 1000)
   })
 }
 

packages/decentraland-ecs/src/cli/wearables.ts

@@ -7,12 +7,10 @@ import { sdk } from '@dcl/schemas'
 
 const serveWearable = ({
   assetJsonPath,
-  baseUrl,
-  catalystRootFolder
+  baseUrl
 }: {
   assetJsonPath: string
   baseUrl: string
-  catalystRootFolder: string
 }) => {
   const wearableDir = path.dirname(assetJsonPath)
   // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -42,8 +40,7 @@ const serveWearable = ({
   const hashedFiles = getFilesFromFolder({
     folder: wearableDir,
     addOriginalPath: false,
-    ignorePattern: ignoreFileContent,
-    rootFolder: catalystRootFolder
+    ignorePattern: ignoreFileContent
   })
 
   const thumbnailFiltered = hashedFiles.filter(
@@ -96,11 +93,9 @@ const serveWearable = ({
 
 export const getAllPreviewWearables = ({
   baseFolders,
-  catalystRootFolder,
   baseUrl
 }: {
   baseFolders: string[]
-  catalystRootFolder: string
   baseUrl: string
 }) => {
   const assetPathArray: string[] = []
@@ -114,7 +109,7 @@ export const getAllPreviewWearables = ({
   const ret = []
   for (const assetJsonPath of assetPathArray) {
     try {
-      ret.push(serveWearable({ assetJsonPath, baseUrl, catalystRootFolder }))
+      ret.push(serveWearable({ assetJsonPath, baseUrl }))
     } catch (err) {
       console.error(
         `Couldn't mock the asset ${assetJsonPath}. Please verify the correct format and scheme.`,
@@ -127,29 +122,26 @@ export const getAllPreviewWearables = ({
 
 export const mockPreviewWearables = (
   app: express.Application,
-  baseFolders: string[],
-  catalystRootFolder: string
+  baseFolders: string[]
 ) => {
-  app.use('/preview-wearables', async (req, res) => {
-    const baseUrl = `http://${req.get('host')}/content/contents`
-    return res.json({
-      ok: true,
-      data: getAllPreviewWearables({ baseUrl, baseFolders, catalystRootFolder })
-    })
-  })
-
   app.use('/preview-wearables/:id', async (req, res) => {
-    const baseUrl = `http://${req.get('host')}/content/contents`
+    const baseUrl = `${req.protocol}://${req.get('host')}/content/contents`
     const wearables = getAllPreviewWearables({
       baseUrl,
-      baseFolders,
-      catalystRootFolder
+      baseFolders
     })
     const wearableId = req.params.id
-
     return res.json({
       ok: true,
       data: wearables.filter((w) => w?.id === wearableId)
     })
   })
+
+  app.use('/preview-wearables', async (req, res) => {
+    const baseUrl = `${req.protocol}://${req.get('host')}/content/contents`
+    return res.json({
+      ok: true,
+      data: getAllPreviewWearables({ baseUrl, baseFolders })
+    })
+  })
 }

packages/decentraland-ecs/src/setupExport.ts

@@ -106,8 +106,7 @@ const setupExport = async ({
         getSceneJson({
           baseFolders: [workDir],
           pointers: sceneJson?.scene?.parcels || ['0,0'],
-          customHashMaker: shaHashMaker,
-          catalystRootFolder: workDir
+          customHashMaker: shaHashMaker
         })
       )
     )
@@ -128,8 +127,7 @@ const setupExport = async ({
       folder: workDir,
       addOriginalPath: true,
       ignorePattern: ignoreFileContent,
-      customHashMaker: shaHashMaker,
-      catalystRootFolder: workDir
+      customHashMaker: shaHashMaker
     })
     if (contentStatic?.content) {
       for (const $ of contentStatic?.content) {

packages/decentraland-ecs/src/setupProxy.ts

@@ -4,6 +4,7 @@ import * as express from 'express'
 import { createStaticRoutes } from './cli/setupUtils'
 import { mockCatalyst } from './cli/mock-catalyst'
 import { mockPreviewWearables } from './cli/wearables'
+import { sdk } from '@dcl/schemas'
 
 const setupProxy = (dcl: any, app: express.Application) => {
   // first resolve all dependencies in the local current working directory
@@ -44,16 +45,38 @@ const setupProxy = (dcl: any, app: express.Application) => {
     })
   )
 
-  const baseSceneFolders: string[] = [dcl.getWorkingDir()]
+  let baseSceneFolders: string[] = [dcl.getWorkingDir()]
+  let baseWearableFolders: string[] = [dcl.getWorkingDir()]
+
+  // TODO: merge types from github.com/decentraland/cli
+  if (dcl.workspace) {
+    const projects = dcl.workspace.getAllProjects()
+    if (!!projects?.length) {
+      const { wearables, scenes } = projects.reduce(
+        (acc: { wearables: string[]; scenes: string[] }, project: any) => {
+          const projectType = project.getInfo().sceneType
+          const projectDir = project.getProjectWorkingDir()
+          if (projectType === sdk.ProjectType.SCENE) acc.scenes.push(projectDir)
+          if (projectType === sdk.ProjectType.PORTABLE_EXPERIENCE)
+            acc.wearables.push(projectDir)
+          return acc
+        },
+        { wearables: [], scenes: [] }
+      )
+
+      baseSceneFolders = scenes
+      baseWearableFolders = wearables
+    }
+  }
+
   try {
-    mockCatalyst(app, baseSceneFolders, dcl.getWorkingDir())
+    mockCatalyst(app, [...baseSceneFolders, ...baseWearableFolders])
   } catch (err) {
     console.error(`Fatal error, couldn't mock the catalyst`, err)
   }
 
-  const baseWearableFolders: string[] = [dcl.getWorkingDir()]
   try {
-    mockPreviewWearables(app, baseWearableFolders, dcl.getWorkingDir())
+    mockPreviewWearables(app, baseWearableFolders)
   } catch (err) {
     console.error(`Fatal error, couldn't mock the wearables`, err)
   }