diff --git a/.gitmodules b/.gitmodules index 6b3475c..a285f59 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = sample url = https://github.com/kristiker/source1import branch = sample +[submodule "utils/shared/keyvalues3"] + path = utils/shared/keyvalues3 + url = https://github.com/kristiker/keyvalues3 + branch = s1 diff --git a/requirements.txt b/requirements.txt index 3ac5998..bbf71a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ numpy vdf dataclassy==0.10.4 bsp_tool==0.3.1 +murmurhash2==0.2.10 # Models/qc-import parsimonious==0.10.0 diff --git a/utils/maps_import.py b/utils/maps_import.py index 4a90d5d..7ae7e98 100644 --- a/utils/maps_import.py +++ b/utils/maps_import.py @@ -1,8 +1,13 @@ +from functools import cache +import shutil +import struct from dataclassy import dataclass, factory + import shared.base_utils2 as sh import vdf import bsp_tool +import shared.keyvalues3 as kv3 import itertools from pathlib import Path import shared.datamodel as dmx @@ -14,22 +19,27 @@ _ElementArray as element_array, _IntArray as int_array, _StrArray as string_array, + _Vector as datamodel_vector_t, ) OVERWRITE_MAPS = False IMPORT_VMF_ENTITIES = True IMPORT_BSP_ENTITIES = False #WRITE_TO_PREFAB = True +IMPORT_BSP_TO_VMAP_C = True maps = Path("maps") mapsrc = Path("mapsrc") -def out_vmap_name(in_vmf: Path): +def out_vmap_name(in_vmf: Path) -> Path: root = mapsrc if in_vmf.local.is_relative_to(mapsrc) else maps return sh.EXPORT_CONTENT / maps / "source1imported" / "entities" / in_vmf.local.relative_to(root).with_suffix(".vmap") +def out_vmap_c_name(in_vmf: Path) -> Path: + return sh.EXPORT_GAME / in_vmf.local.with_suffix(".vmap_c") + def main(): - + if IMPORT_VMF_ENTITIES: print("Importing vmf entities!") for vmf_path in itertools.chain( @@ -37,14 +47,393 @@ def main(): sh.collect(maps, ".vmf", ".vmap", OVERWRITE_MAPS, out_vmap_name) ): ImportVMFEntitiesToVMAP(vmf_path) - + if IMPORT_BSP_ENTITIES: print("Importing bsp entities!") for bsp_path in sh.collect(maps, ".bsp", ".vmap", OVERWRITE_MAPS, out_vmap_name): ImportBSPEntitiesToVMAP(bsp_path) + if IMPORT_BSP_TO_VMAP_C: + print("Converting bsp to vpk!") + for bsp_path in sh.collect(maps, ".bsp", ".vpk", OVERWRITE_MAPS, out_vmap_c_name): + ImportBSPToVPK(bsp_path) + print("Looks like we are done!") +import shared.worldnode as wnod +import shared.world as wrld +import shared.entities as entities +import shared.physics as physics +from murmurhash2 import murmurhash2 + +def ImportBSPToVPK(bsp_path: Path): + compiled_vmap_path = out_vmap_c_name(bsp_path) + compiled_lumps_folder = compiled_vmap_path.parent / compiled_vmap_path.stem + compiled_lumps_folder.parent.MakeDir() + + sh.status(f'- Reading {bsp_path.local}') + bsp: bsp_tool.ValveBsp = bsp_tool.load_bsp(bsp_path.as_posix()) + + sprp_lump = sprp(bsp.GAME_LUMP.sprp) + _dprp: bsp_tool.base.lumps.RawBspLump = bsp.GAME_LUMP.dprp + + import numpy as np + from math import sin, cos + def transforms_to_3x4(origin: vector3, angles: qangle) -> list: + m = np.zeros((3, 4)) + # TODO: Figure out rotation + m[:, 3] = origin + #return m.tolist() + α = angles[1] + β = angles[0] + γ = angles[2] + return [ + [cos(α)*cos(β), cos(α)*sin(β)*sin(γ)-sin(α)*cos(γ), cos(α)*sin(β)*cos(γ)+sin(α)*sin(γ), origin[0]], + [sin(α)*cos(β), sin(α)*sin(β)*sin(γ)+cos(α)*cos(γ), sin(α)*sin(β)*cos(γ)-cos(α)*sin(γ), origin[1]], + [-sin(β), cos(β)*sin(γ), cos(β)*cos(γ), origin[2]], + ] + + def col32_to_vec4(diffuse: int) -> list: + return [ + (diffuse & 0xFF) / 255, + ((diffuse >> 8) & 0xFF) / 255, + ((diffuse >> 16) & 0xFF) / 255, + ((diffuse >> 24) & 0xFF) / 255, + ] + + worldnode000 = wnod.WorldNode() + worldnode000.m_boundsGroups.append(wnod.Bounds([-9999.0, -9999.0, -9999.0], [9999999.0, 9999999.0, 9999999.0])) + + worldnode000.add_to_layer(wnod.SceneObject( + m_nObjectID = 0, + m_vTransform = transforms_to_3x4([0, 0, 0], [0, 0, 0]), + m_flFadeStartDistance=0, + m_flFadeEndDistance=0, + m_vTintColor=[1, 1, 1, 1], + m_skin="", + m_nObjectTypeFlags=0, + m_vLightingOrigin=[0, 0, 0], + m_nBoundsGroupIndex=0, + m_renderableModel = kv3.flagged_value("maps/ar_lunacy/worldnodes/bsp.vmdl", kv3.Flag.resource), + ), + "world_layer_base" + ) + + for static_prop in sprp_lump.static_props: + model_path = Path(sprp_lump.model_names[static_prop.PropType]).with_suffix(".vmdl") + prop_sceneobject = wnod.SceneObject( + m_nObjectID = 0, + m_vTransform = transforms_to_3x4(static_prop.Origin, static_prop.Angles), + m_flFadeStartDistance = static_prop.FadeMinDist, + m_flFadeEndDistance = static_prop.FadeMaxDist, + m_vTintColor = col32_to_vec4(static_prop.DiffuseModulation), + m_skin = str(static_prop.Skin), + m_nObjectTypeFlags = static_prop.Flags, + m_vLightingOrigin = static_prop.LightingOrigin, # TODO: relative to origin? + m_nBoundsGroupIndex = 0, + m_renderableModel = kv3.flagged_value(model_path.as_posix(), kv3.Flag.resource), + ) + + worldnode000.add_to_layer(prop_sceneobject, "world_layer_base") + + + worldnode000_path = Path(r"D:\Users\kristi\Documents\s&box projects\cs\maps\ar_lunacy\worldnodes") / "node000.vwnod_c" + worldnode000_path.parent.MakeDir() + + def write_resource_data_by_template(data: object, template_path: Path, resurce_path: Path): + # TODO: Resource external references + resource = bytearray(template_path.read_bytes()) + DATA = bytes(kv3.binarywriter.BinaryLZ4(kv3.KV3File(data))) + # adjust block size bytes + blocksize_location = resource.find(b"DATA") + 8 + resource = resource[:blocksize_location] + struct.pack("> 8) & 0xFFFFFF + + +@dataclass +class compactledgenode_t(_common_struct): + offset_right_node: ctypes.c_int + offset_compact_ledge: ctypes.c_int + center: vector3 + radius: ctypes.c_float + box_size_x: ctypes.c_ubyte + box_size_y: ctypes.c_ubyte + box_size_z: ctypes.c_ubyte + free_0: ctypes.c_ubyte + + @property + def is_terminal(self): + return self.offset_right_node == 0 + +@dataclass +class compactledge_t(_common_struct): + c_point_offset: ctypes.c_int + ledgetree_node_offset: ctypes.c_int | ctypes.c_int + bitfield0: ctypes.c_int + n_triangles: ctypes.c_short + for_future_use: ctypes.c_short + +@dataclass +class compacttriangle_t(_common_struct): + bitfield0: ctypes.c_int + compact_edge_0: ctypes.c_int + compact_edge_1: ctypes.c_int + compact_edge_2: ctypes.c_int + + tri_index = property(lambda self: self.bitfield0 & 0xFFF) + pierce_index = property(lambda self: (self.bitfield0 >> 12) & 0xFFF) + material_index = property(lambda self: (self.bitfield0 >> 24) & 0x7F) + is_virtual = property(lambda self: (self.bitfield0 >> 31) & 0x1) + + compact_edge_0_start_point_index = property(lambda self: self.compact_edge_0 & 0xFFFF) + compact_edge_0_opposite_index = property(lambda self: (self.compact_edge_0 >> 16) & 0x7FFF) + compact_edge_0_is_virtual = property(lambda self: (self.compact_edge_0 >> 31) & 0x1) + compact_edge_1_start_point_index = property(lambda self: self.compact_edge_1 & 0xFFFF) + compact_edge_1_opposite_index = property(lambda self: (self.compact_edge_1 >> 16) & 0x7FFF) + compact_edge_1_is_virtual = property(lambda self: (self.compact_edge_1 >> 31) & 0x1) + compact_edge_2_start_point_index = property(lambda self: self.compact_edge_2 & 0xFFFF) + compact_edge_2_opposite_index = property(lambda self: (self.compact_edge_2 >> 16) & 0x7FFF) + compact_edge_2_is_virtual = property(lambda self: (self.compact_edge_2 >> 31) & 0x1) + def ImportVMFEntitiesToVMAP(vmf_path): vmap_path = out_vmap_name(vmf_path) @@ -276,5 +665,5 @@ def main_to_root(vmap, main_key: str, sub): vmap[replacement] = _type(sub[t]) if __name__ == '__main__': - sh.parse_argv() + sh.parse_argv(globals()) main() diff --git a/utils/shared/base_utils2.py b/utils/shared/base_utils2.py index 856905d..c8a28eb 100644 --- a/utils/shared/base_utils2.py +++ b/utils/shared/base_utils2.py @@ -82,6 +82,7 @@ def __call__(self, args: list[str] = []) -> Optional[subprocess.CompletedProcess dmxconvert = auto() resourcecompiler = auto() resourcecopy = auto() + vpk = auto() class eEngineFolder(Enum): "Source 2 main folders relative to root" diff --git a/utils/shared/datamodel.py b/utils/shared/datamodel.py index 5c3a84a..d912b02 100644 --- a/utils/shared/datamodel.py +++ b/utils/shared/datamodel.py @@ -161,10 +161,13 @@ class _FloatArray(_Array): class _StrArray(_Array): type = str +import types class _Vector(list): type = None type_str = "" def __init__(self,l): + if isinstance(l, types.GeneratorType): + l = list(l) if len(l) != len(self.type_str): raise TypeError("Expected {} values, got {}".format(len(self.type_str), len(l))) l = _validate_array_list(l,float if self.type is None else self.type) diff --git a/utils/shared/entities.py b/utils/shared/entities.py new file mode 100644 index 0000000..3b96654 --- /dev/null +++ b/utils/shared/entities.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass, field + +@dataclass +class Entity: + m_keyValuesData: bytearray + m_connections: list + +@dataclass +class Ents: + m_name: str = "default_ents" + m_hammerUniqueId: str = "" + m_flags: str = "ENTITY_LUMP_NONE" + m_manifestName: str = "" + m_childLumps: list[str] = field(default_factory=list) + m_entityKeyValues: list[Entity] = field(default_factory=list) diff --git a/utils/shared/keyvalues3 b/utils/shared/keyvalues3 new file mode 160000 index 0000000..99687b1 --- /dev/null +++ b/utils/shared/keyvalues3 @@ -0,0 +1 @@ +Subproject commit 99687b1db8b709396ceb2a2e055677186f093a35 diff --git a/utils/shared/keyvalues3.py b/utils/shared/keyvalues3.py.old similarity index 94% rename from utils/shared/keyvalues3.py rename to utils/shared/keyvalues3.py.old index 04e8a41..ebef493 100644 --- a/utils/shared/keyvalues3.py +++ b/utils/shared/keyvalues3.py.old @@ -1,5 +1,5 @@ from pathlib import Path -from dataclasses import dataclass +from dataclasses import asdict, dataclass, is_dataclass from uuid import UUID @dataclass(frozen=True) @@ -20,7 +20,10 @@ def __str__(self): class KV3File(dict): def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + if len(args) and is_dataclass(args[0]): + super().__init__(asdict(args[0])) + else: + super().__init__(*args, **kwargs) self.header = KV3Header(format="source1imported") def __str__(self): diff --git a/utils/shared/maps/default_ents.vents_c.template b/utils/shared/maps/default_ents.vents_c.template new file mode 100644 index 0000000..a1d2822 Binary files /dev/null and b/utils/shared/maps/default_ents.vents_c.template differ diff --git a/utils/shared/maps/node000.vrman_c b/utils/shared/maps/node000.vrman_c new file mode 100644 index 0000000..b478ddc Binary files /dev/null and b/utils/shared/maps/node000.vrman_c differ diff --git a/utils/shared/maps/node000.vwnod_c.template b/utils/shared/maps/node000.vwnod_c.template new file mode 100644 index 0000000..ab2e7ea Binary files /dev/null and b/utils/shared/maps/node000.vwnod_c.template differ diff --git a/utils/shared/maps/world_physics.vphys_c.template b/utils/shared/maps/world_physics.vphys_c.template new file mode 100644 index 0000000..5832a9a Binary files /dev/null and b/utils/shared/maps/world_physics.vphys_c.template differ diff --git a/utils/shared/physics.py b/utils/shared/physics.py new file mode 100644 index 0000000..b7ee4fb --- /dev/null +++ b/utils/shared/physics.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass, field + +from .worldnode import Bounds + +@dataclass +class _BaseShape: + m_nCollisionAttributeIndex: int = 0 + m_nSurfacePropertyIndex: int = 0 + m_UserFriendlyName: str = "" + +@dataclass +class Hull(_BaseShape): + @dataclass + class Hull: + @dataclass + class Plane: + m_vNormal: list[float] + m_flOffset: float + @dataclass + class Edge: + m_nNext: int + m_nTwin: int + m_nOrigin: int + m_nFace: int + @dataclass + class Face: + m_nEdge: int + m_vCentroid: list[float] = field(default_factory=list) + m_flMaxAngularRadius: float = 0.0 + m_Vertices: list[list[float]] = field(default_factory=list) + m_Planes: list[Plane] = field(default_factory=list) + m_Edges: list[Edge] = field(default_factory=list) + m_Faces: list[Face] = field(default_factory=list) + m_vOrthographicAreas: list[float] = field(default_factory=list) + m_MassProperties: list[float] = field(default_factory=list) + m_flVolume: float = 0.0 + m_flMaxMotionRadius: float = 0.0 + m_flMinMotionThickness: float = 0.0 + m_Bounds: Bounds = field(default_factory=Bounds) + m_nFlags: int = 0 + m_Hull: Hull = field(default_factory=Hull) + + +@dataclass +class Mesh(_BaseShape): + @dataclass + class Mesh: + m_vMin: list[float] = field(default_factory=list) + m_vMax: list[float] = field(default_factory=list) + m_Materials: list[int] = field(default_factory=list) + m_vOrthographicAreas: list[float] = field(default_factory=list) + m_Nodes: bytearray = field(default_factory=bytearray) + m_Triangles: bytearray = field(default_factory=bytearray) + m_Vertices: bytearray = field(default_factory=bytearray) + m_Mesh: Mesh =field(default_factory=Mesh) + +@dataclass +class Shape: + m_spheres: list = field(default_factory=list) + m_capsules: list = field(default_factory=list) + m_hulls: list[Hull] = field(default_factory=list) + m_meshes: list[Mesh] = field(default_factory=list) + + +@dataclass +class Part: + m_nFlags: int + m_flMass: float + """mas in kg""" + m_rnShape: Shape + m_CollisionAttributeIndices: list[int] = field(default_factory=list) + m_nSurfacepropertyIndices: list = field(default_factory=list) + m_nCollisionAttributeIndex: int = 0 + m_nReserved: int = 0 + m_flInertiaScale: float = 1.0 + m_flLinearDamping: float = 0.0 + m_flAngularDamping: float = 0.0 + m_bOverrideMassCenter: bool = False + m_vMassCenterOverride: list[float] = field(default_factory=list) + +@dataclass +class CollisionAttribute: + m_CollisionGroup: int + m_InteractAs: list[int] + m_InteractWith: list[int] + m_InteractExclude: list[int] + m_CollisionGroupString: str = "Default" + m_InteractAsStrings: list[str] = field(default_factory=list) + m_InteractWithStrings: list[str] = field(default_factory=list) + m_InteractExcludeStrings: list[str] = field(default_factory=list) + +@dataclass +class PhysX: + m_nFlags: int = 0 + m_nRefCounter: int = 0 + m_bonesHash: list = field(default_factory=list) + m_boneNames: list = field(default_factory=list) + m_indexNames: list = field(default_factory=list) + m_indexHash: list = field(default_factory=list) + m_bindPose: list = field(default_factory=list) + m_parts: list[Part] = field(default_factory=list) + m_constraints2: list = field(default_factory=list) + m_joints: list = field(default_factory=list) + m_pFeModel: object = None + m_boneParents: list = field(default_factory=list) + m_surfacePropertyHashes: list[int] = field(default_factory=list) + m_collisionAttributes: list[CollisionAttribute] = field(default_factory=list) + m_debugPartNames: list = field(default_factory=list) + m_embeddedKeyvalues: str = "" \ No newline at end of file diff --git a/utils/shared/world.py b/utils/shared/world.py new file mode 100644 index 0000000..84745c1 --- /dev/null +++ b/utils/shared/world.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass, field + +@dataclass +class BuilderParams: + m_nSizeBytesPerVoxel: int + m_flMinDrawVolumeSize: float + m_flMinDistToCamera: float + m_flMinAtlasDist: float + m_flMinSimplifiedDist: float + m_flHorzFOV: float + m_flHalfScreenWidth: float + m_nAtlasTextureSizeX: int + m_nAtlasTextureSizeY: int + m_nUniqueTextureSizeX: int + m_nUniqueTextureSizeY: int + m_nCompressedAtlasSize: int + m_flGutterSize: float + m_flUVMapThreshold: float + m_vWorldUnitsPerTile: list[float] #= field(default_factory=lambda:[10000.000000, 10000.000000, 1000.000000]) + m_nMaxTexScaleSlots: int + m_bWrapInAtlas: bool + m_bBuildBakedLighting: bool + m_vLightmapUvScale: list[float] #= field(default_factory=lambda:[1.0, 1.0]) + m_nCompileTimestamp: int = 1657194806 + m_nCompileFingerprint: int = 8654431948308770350 + +@dataclass +class Node: + m_Flags: int # flags + m_nParent: int + m_vOrigin: list[float] = field(default_factory=lambda:[0.0, 0.0, 0.0]) # vec3 + m_vMinBounds: list[float] = field(default_factory=list) # vec3 + m_vMaxBounds: list[float] = field(default_factory=list) # vec3 + m_flMinimumDistance: float = -1.000000 + m_ChildNodeIndices: list[int] = field(default_factory=list) + m_worldNodePrefix: str = "" # maps\map\worldnodes\node000 + +@dataclass +class LightingInfo: + m_nLightmapVersionNumber: int = 8 + m_nLightmapGameVersionNumber: int = 1 + m_vLightmapUvScale: list[float] = field(default_factory=lambda:[1.0, 1.0]) + m_bHasLightmaps: bool = True + m_lightMaps: list[str] = field(default_factory=list) + #[ + # resource:"maps/map/lightmaps/direct_light_indices.vtex", + # resource:"maps/map/lightmaps/direct_light_strengths.vtex", + # resource:"maps/map/lightmaps/irradiance.vtex", + # resource:"maps/map/lightmaps/directional_irradiance.vtex", + # resource:"maps/map/lightmaps/debug_chart_color.vtex", + #] + +@dataclass +class World: + """Format for Valve vwrld_c files""" + m_builderParams: BuilderParams + m_worldNodes: list[Node] = field(default_factory=list) + m_worldLightingInfo: LightingInfo = field(default_factory=LightingInfo) + m_entityLumps: list[str] = field(default_factory=list) \ No newline at end of file diff --git a/utils/shared/worldnode.py b/utils/shared/worldnode.py new file mode 100644 index 0000000..17a533b --- /dev/null +++ b/utils/shared/worldnode.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass, field + +@dataclass +class SceneObject: + m_nObjectID: int + m_vTransform: list[list[float]] + m_flFadeStartDistance: float + m_flFadeEndDistance: float + m_vTintColor: list[float] + m_skin: str + m_nObjectTypeFlags: int | str + """"int flags or flag name""" + m_vLightingOrigin: list[float] + m_nLightGroup: int = 0 + m_nOverlayRenderOrder: int = 0 + m_nLODOverride: int = -1 + m_nCubeMapPrecomputedHandshake: int = 0 + m_nLightProbeVolumePrecomputedHandshake: int = 0 + m_nBoundsGroupIndex: int = -1 + m_renderableModel: str = None # resource flag + """vmdl file to render""" + m_renderable: str = None + """vmesh file to render""" + #m_externalTextures: list[str] = field(default_factory=list) + #m_VisClusterMemberBits: int = 0 + +@dataclass +class Bounds: + m_vMinBounds: list[float] + m_vMaxBounds: list[float] + +@dataclass +class MaterialOverride: + m_nSceneObjectIndex: int + m_nSubSceneObject: int + m_nDrawCallIndex: int + m_pMaterial: str # resource flag + +@dataclass +class NodeLightingInfo: + m_nLightmapVersionNumber: int = 0 + m_nLightmapGameVersionNumber: int = 0 + m_vLightmapUvScale: list = field(default_factory=lambda: [1.0, 1.0]) + m_bHasLightmaps: bool = False + m_lightMaps: list = field(default_factory=list) + +@dataclass +class WorldNode: + """Format for Valve vwnod_c files""" + m_sceneObjects: list[SceneObject] = field(default_factory=list) + m_infoOverlays: list = field(default_factory=list) + m_visClusterMembership: list = field(default_factory=list) + m_boundsGroups: list[Bounds] = field(default_factory=list) + m_boneOverrides: list = field(default_factory=list) + m_aggregateSceneObjects: list = field(default_factory=list) + m_extraVertexStreamOverrides: list = field(default_factory=list) + m_materialOverrides: list[MaterialOverride] = field(default_factory=list) + m_extraVertexStreams: list = field(default_factory=list) + m_layerNames: list[str] = field(default_factory=list) + m_sceneObjectLayerIndices: list[int] = field(default_factory=list) + m_overlayLayerIndices: list = field(default_factory=list) + m_grassFileName: str = "" + m_nodeLightingInfo: NodeLightingInfo = field(default_factory=NodeLightingInfo) + + def add_to_layer(self, scene_object: SceneObject, layer: str): + """Add a scene object to a layer""" + if layer not in self.m_layerNames: + self.m_layerNames.append(layer) + self.m_sceneObjectLayerIndices.append(self.m_layerNames.index(layer)) + self.m_sceneObjects.append(scene_object)