diff --git a/src/Flex.tsx b/src/Flex.tsx
index f017bf0..c173097 100644
--- a/src/Flex.tsx
+++ b/src/Flex.tsx
@@ -1,6 +1,6 @@
 import React, { useLayoutEffect, useMemo, useCallback, PropsWithChildren, useRef } from 'react'
 import Yoga, { YogaNode } from 'yoga-layout-prebuilt'
-import { Vector3, Group, Box3 } from 'three'
+import { Vector3, Group, Box3, Object3D } from 'three'
 import { useFrame, useThree, ReactThreeFiber } from '@react-three/fiber'
 
 import { setYogaProperties, rmUndefFromObj, vectorFromObject, Axis, getDepthAxis, getFlex2DSize } from './util'
@@ -22,6 +22,18 @@ export type FlexProps = PropsWithChildren<
     R3FlexProps &
     Omit<ReactThreeFiber.Object3DNode<THREE.Group, typeof Group>, 'children'>
 >
+interface BoxesItem {
+  node: YogaNode
+  group: Group
+  flexProps: R3FlexProps
+  centerAnchor: boolean
+}
+
+// This is not very performant
+// We should probably optimize it, options are
+// * Memoization
+// * Precalculation of this when registering a box
+const hasBoxChildren = (boxes: BoxesItem[], children: Object3D[]) => boxes.some(({ group }) => children.includes(group))
 
 /**
  * Flex container. Can contain Boxes
@@ -198,7 +210,7 @@ export function Flex({
   ])
 
   // Keeps track of the yoga nodes of the children and the related wrapper groups
-  const boxesRef = useRef<{ node: YogaNode; group: Group; flexProps: R3FlexProps; centerAnchor: boolean }[]>([])
+  const boxesRef = useRef<BoxesItem[]>([])
   const registerBox = useCallback(
     (node: YogaNode, group: Group, flexProps: R3FlexProps, centerAnchor: boolean = false) => {
       const i = boxesRef.current.findIndex((b) => b.node === node)
@@ -272,7 +284,7 @@ export function Flex({
           // Forced size, no need to calculate bounding box
           node.setWidth(scaledWidth)
           node.setHeight(scaledHeight)
-        } else {
+        } else if (!hasBoxChildren(boxesRef.current, group.children)) {
           // No size specified, calculate bounding box
           boundingBox.setFromObject(group).getSize(vec)
           node.setWidth(scaledWidth || vec[mainAxis] * scaleFactor)