python scripts

[duncan]

Hi,

I wrote a set of python scripts that import building (WHP) and moth (SHP) models, plus converts textures (SPR) to BMP from hardwar data files into blender. At the moment I have little time to continue work on them so I thought about dumping them somewhere online. What would be a good way to release? Shall I copy paste into the forum? Can I have a scripts section on the forum? Then each script can have its own thread and be discussed upon, explained and further developed. Or shall I stick everything in a zip and upload someplace. Anyone interested can also pm me and I can send by email.

I tried to keep the scripts setup flexible so it can partly be reused to convert data to other destinations.
Cheers,
Duncan

[rotary]

You've got my email-adress! ;D

[duncan]

Jul 29, 2011 4:11:10 GMT -5 rotary said:

You've got my email-adress! ;D

But you already have the scripts.

[duncan]

Ok, i dumped the scripts on sourceforge.

sourceforge.net/p/hwpython/

[berylraven]

Hey Duncan, great work on these scripts! All of it worked great on windows once you get the right path format. (not used to coding in windows). I was wondering if you had finished the moth2blend section of code and its dependencies whpb2 and whpconvert2 since they are not up on sourceforge and if they are could you send them to me or post them there? buildings and environment are great but i am much more interested in playing around with the moths. Thanks for making these cause they saved me a TON of time. :)

[duncan]

Mar 21, 2012 20:46:18 GMT -5 berylraven said:

Hey Duncan, great work on these scripts! All of it worked great on windows once you get the right path format. (not used to coding in windows). I was wondering if you had finished the moth2blend section of code and its dependencies whpb2 and whpconvert2 since they are not up on sourceforge and if they are could you send them to me or post them there? buildings and environment are great but i am much more interested in playing around with the moths. Thanks for making these cause they saved me a TON of time. :)

Its been a while, i will have a look. PM your email address in the meantime then I can sent them once I found them.

Or copy paste them from below. It seems I missed three files in the upload - or used the old versions for them. Also uploaded to scourceforge.

hwwhp2.py
whpconvert2.py
whpb2.py

Btw thanks and glad you were able to get the scripts working.

[duncan]


#whpb2.py
#a container for whp data to be shared by multiple modules

global whptextures
global whptexids
global facevertices
global facetexture
global facetcoords
global whpvnum
global whpvertices
global whpscaling

global bvertices
global bfaces
global btextures
global bfacematid
global bfaceuv

whpvnum     = 0
whpscaling  = 0
whpvertices = []
facevertices= []
facetexture = []
facetcoords = []
whptextures = []
whptexids   = []

btexids    = []
btextures  = []
bvertices  = []
bfaces     = []
bfaceuv    = []
bfacemat   = []
bfacematid = []


[duncan]


hwwhp2.py
#Python 3.2 (r32:88452, Feb 20 2011, 10:19:59)

#Duncan June 2011
#hwwhp.py
#initiated for use in res2blend
#
#to unlock data in whp file structure
#cleaned up code dealing with faces a bit from earlier version


import array
from struct import unpack_from

from hwpaths import *
import hwres
import whpb2


#offset to whp data in wxp datablock
#wxp offset in res obtained separately
def openwhp(whpoffset,whpsize):

   global whptextures
   global whpvertices
   global whptexids
   global facevertices
   global facetexture
   global facetcoords
   global whpvnum
   global whpvtuple
   global whpscaling
   global bvertices
   global bfaces
   global btextures
   global bfacemat
   global bfacematid
   global bfaceuv
   global bfacetname

   wxpindex  = hwres.findfileindex('HARDWAR.WXP')
   wxpoffset = hwres.getfileoffset(wxpindex)
   wxpsize   = hwres.getfilesize(wxpindex)
   
   f = open(respath+resname,'rb')

   dump = f.seek(wxpoffset+whpoffset+20,0)

   whp=array.array('B')
   whp.fromfile(f,whpsize)

   f.close()


   #grab header
   bytemaph = 'HHHHLLLLLLllllllL'
   whpheader = unpack_from(bytemaph,whp,0)
   whpversion = whpheader[0]
   whpscaling = whpheader[1]
   whpruleset = whpheader[2]
   whpsize = whpheader[4]
   whpvoffset = whpheader[6]
   whpfoffset = whpheader[7]
   whptoffset = whpheader[8]
   whpbsphere = whpheader[9]
   whpbboxx1 = whpheader[10]
   whpbboxy1 = whpheader[11]
   whpbboxz1 = whpheader[12]
   whpbboxx2 = whpheader[13]
   whpbboxy2 = whpheader[14]
   whpbboxz2 = whpheader[15]

   
   whpvnum = 0
   whpfnum = 0
   whptnum = 0
   temptuple = unpack_from('H',whp,whpvoffset)
   whpvnum = temptuple[0]
   temptuple = unpack_from('H',whp,whpfoffset)
   whpfnum = temptuple[0]
   temptuple = unpack_from('H',whp,whptoffset)
   whptnum = temptuple[0]


   # grab faces

   faceoffset = []
   facenormal = []
   facetype   = []
   facetypesub= []
   facevnum   = []
   facetexture= []
   facenormal = []
   facevertices=[]
   facetcoords= []
   facevnormals=[]
   
   ftype = []
   ftypesub = []

   faceoffset.append(whpfoffset+2)
   for i in range(whpfnum):
      ftype.append(whp[faceoffset[i]+1])
      ftypesub.append(whp[faceoffset[i]+11])

      facetexture.append(whp[faceoffset[i]+2])
      fnormal = unpack_from('hhh',whp,faceoffset[i]+4)
      facenormal.append(list(fnormal))          

      if   ftype[i] == 0x04:       #untextured triangle
         facevnum.append(3)
         faceoffset.append(faceoffset[i]+32)
         fvertices = unpack_from('BBB',whp,faceoffset[i]+10)
         facevertices.append(list(fvertices))
         facetcoords.append([])
         fvnormals = unpack_from('hhhhhhhhh',whp,faceoffset[i]+14)
         facevnormals.append(list(fvnormals))

      elif ftype[i] == 0x08:       #  textured triangle
         facevnum.append(3)
         faceoffset.append(faceoffset[i]+44)
         fvertices = unpack_from('BBB',whp,faceoffset[i]+10)
         facevertices.append(list(fvertices))
         ftcoords  = unpack_from('HHHHHH',whp,faceoffset[i]+14)
         facetcoords.append(list(ftcoords))
         fvnormals = unpack_from('hhhhhhhhh',whp,faceoffset[i]+26)
         facevnormals.append(list(fvnormals))

      elif ftype[i] == 0x40:       #untextured quad
         facevnum.append(4)
         faceoffset.append(faceoffset[i]+38)
         fvertices = unpack_from('BBBB',whp,faceoffset[i]+10)
         facevertices.append(list(fvertices))
         facetcoords.append([])
         fvnormals = unpack_from('hhhhhhhhhhhh',whp,faceoffset[i]+14)
         facevnormals.append(list(fvnormals))

      elif ftype[i] == 0x80:       #  textured quad
         facevnum.append(4)
         faceoffset.append(faceoffset[i]+54)
         fvertices = unpack_from('BBBB',whp,faceoffset[i]+10)
         facevertices.append(list(fvertices))
         ftcoords  = unpack_from('HHHHHHHH',whp,faceoffset[i]+14)
         facetcoords.append(list(ftcoords))
         fvnormals = unpack_from('hhhhhhhhhhhh',whp,faceoffset[i]+30)
         facevnormals.append(list(fvnormals))

      elif ftype[i] == 0x00:
         if   ftypesub[i] == 0x40: #untextured polygon

            facevnum.append(whp[faceoffset[i]+10])
            faceoffset.append(faceoffset[i]+12+facevnum[i]*8)

            facetcoords.append([])

            bytemapstring = ''
            for ii in range(facevnum[i]): bytemapstring += 'H'
            fvertices = unpack_from(bytemapstring,whp,faceoffset[i]+12)
            facevertices.append(list(fvertices))

            bytemapstring = ''
            for ii in range(facevnum[i]): bytemapstring += 'hhh'
            fvnormals = unpack_from(bytemapstring,whp,faceoffset[i]+12+facevnum[i]*2)
            facevnormals.append(list(fvnormals))

         elif ftypesub[i] == 0xC0: #  textured polygon

            facevnum.append(whp[faceoffset[i]+10])
            faceoffset.append(faceoffset[i]+12+facevnum[i]*12)

            bytemapstring = ''
            for ii in range(facevnum[i]): bytemapstring += 'H'
            fvertices = unpack_from(bytemapstring,whp,faceoffset[i]+12)
            facevertices.append(list(fvertices))

            bytemapstring = ''
            for ii in range(facevnum[i]): bytemapstring += 'hhh'
            fvnormals = unpack_from(bytemapstring,whp,faceoffset[i]+12+facevnum[i]*2)
            facevnormals.append(list(fvnormals))

            bytemapstring = ''
            for ii in range(facevnum[i]): bytemapstring += 'HH'
            ftcoords = unpack_from(bytemapstring,whp,faceoffset[i]+12+facevnum[i]*2+facevnum[i]*3*2)
            facetcoords.append(list(ftcoords))

         else: print('unrecognised polygon type')
      else: print('unrecognised face type')


   #vertices
   #unpack_from returns a tuple, which is an unmutable variable, so cannot be calculated upon
   #converted to list

   whpvertices = [[] for i in range(whpvnum)]
   for i in range(whpvnum):
      temptuple = unpack_from('hhh',whp,whpvoffset+2+i*6)
      whpvertices[i] = list(temptuple)


   #textures
   whptextures = [[] for i in range(whptnum)]
   whptexids = [[] for i in range(whptnum)]
   for i in range(whptnum):

      whptexnameoffset = whptoffset+2+i*18
      whptextures[i] = whp[whptexnameoffset:whptexnameoffset+16].tostring()
      whptexids[i] = whp[whptexnameoffset+16]


   #cleanup whptextures, needed since python3.2
   for i in range(len(whptextures)):
      whptextures[i] = str(whptextures[i])
      whptextures[i] = str(whptextures[i][2:])
      whptextures[i] = whptextures[i][:len(whptextures[i])-1]


   whpb2.whpvnum     = whpvnum
   whpb2.whpscaling  = whpscaling
   whpb2.whpvertices = whpvertices
   whpb2.facevertices= facevertices
   whpb2.facetexture = facetexture
   whpb2.facetcoords = facetcoords
   whpb2.whptextures = whptextures
   whpb2.whptexids   = whptexids


[duncan]


#whpconvert2.py

#Python 3.2 (r32:88452, Feb 20 2011, 10:19:59)
#Blender 2.5 hardwar native WHP importer
#
#Duncan June 2011
#adapting for use in res2blend

import array
import math

import whpb2





#-------------------------------------
## Convert whp data for blender import
# - only triangles and quads supported by Blender.Mesh, no larger polygons
# - reversed handedness, since blender is opengl and hardwar is directx
# - blender doesnt use vertex index 0 for faces, starts at 1
# - some scaling needed
# - swap y and z coordinates of vertices, blender uses z as up.
# - recalculate uv coordinates
# - give uv coordinates where none on whpface (airlockdoor)
# - dont forget to follow revese handedness with uv coordinates
# - skip import dummy faces - seem to be double faces, causes flicker


def convertwhp(bscale):

global bvertices
global bfaces
global btextures
#global btexid
global bfacemat
global bfacematid
global bfaceuv
global bfacetname

##scale vertices, plus swap y and z, plus start at index 1 not 0

whpscalefactor = math.pow(2,whpb2.whpscaling)
#bscale = 20000

bvertices = []
bvertices.append([0,0,0]) #vert 0 unused by blender
for i in whpb2.whpvertices:
    #swap y and z
    bvertices.append([i[0],i[2],i[1]])

for i in range(len(bvertices)):
    #apply model scale
    for ii in range(3):
         bvertices[i][ii] *= whpscalefactor

    #apply world scale to fit in blender    
    for ii in range(3):
         bvertices[i][ii] *= 1.0
         bvertices[i][ii] /= bscale


#convert faces
# simple triangulate polygons (fan-wise)
# strip duplicate faces (for example 179524,1848 has a triple face, one dummy, two normal tex)
# duplicate faces cause flicker in blender

#identify double faces
#shift vertices order to smallest first
tempfaces = []
for i in whpb2.facevertices:
   tempfaces.append([[] for ii in range(len(i))])
   smallestvertat = 0
   smallestvertid = i[0]
   for ii in range(len(i)):
      if i[ii] < smallestvertid:
         smallestvertid = i[ii]
         smallestvertat = ii
   for ii in range(len(i)):
      tempfaces[len(tempfaces)-1][ii] = \
          i[smallestvertat + ii - len(i)*math.ceil((0.1+ii+smallestvertat-len(i))/len(i))]

#flag duplicate faces for skip
flagskipface = [0 for iii in range(len(tempfaces))]
for i in range(len(tempfaces)):
   for ii in range(len(tempfaces)-i):
      if tempfaces[i] == tempfaces[len(tempfaces)-ii-1] and i != len(tempfaces)-ii-1:
          #flag preferably dummy as dupicate face
          if whpb2.facetexture[i] == 0: flagskipface[i] = 1
          else: flagskipface[len(tempfaces)-ii-1] = 1


bfaces = []
bfaceuv = []
bfacemat = []
for i in range(len(whpb2.facevertices)):

 # skip flagged faces
 if flagskipface[i] == 1:
    pass
 else:

   # polygon triangulation (triangle fan)
   if len(whpb2.facevertices[i]) > 4:
      for ii in range(len(whpb2.facevertices[i])-2):
         bfaces.append([whpb2.facevertices[i][0], \
                        whpb2.facevertices[i][ii+1], \
                        whpb2.facevertices[i][ii+2]])
         bfacemat.append(whpb2.facetexture[i])

         if len(whpb2.facetcoords[i]) > 0:
            bfaceuv.append([whpb2.facetcoords[i][0], \
                            whpb2.facetcoords[i][1], \
                            whpb2.facetcoords[i][ii*2+2], \
                            whpb2.facetcoords[i][ii*2+3], \
                            whpb2.facetcoords[i][ii*2+4], \
                            whpb2.facetcoords[i][ii*2+5]])

   else:
      bfaces.append(whpb2.facevertices[i])
      bfaceuv.append(whpb2.facetcoords[i])
      bfacemat.append(whpb2.facetexture[i])


#add 1 to vertex indices
for i in range(len(bfaces)):
    for ii in range(len(bfaces[i])):
        bfaces[i][ii] += 1

#reverse handedness face vertices
for i in range(len(bfaces)):
    bfaces[i].reverse()

#reverse handedness uv coordinate (pairwise)
for i in range(len(bfaceuv)):
    tempuv = int(len(bfaceuv[i])/2)
    tempuva = [[] for ii in range(tempuv)]
    tempuvb = [[] for ii in range(tempuv)]
    for ii in range(tempuv):
         tempuva[ii] = bfaceuv[i][ii*2]
         tempuvb[ii] = bfaceuv[i][ii*2+1]
    tempuva.reverse()
    tempuvb.reverse()
    for ii in range(tempuv):
         bfaceuv[i][ii*2] = tempuva[ii]
         bfaceuv[i][ii*2+1] = tempuvb[ii]


#convert uv coords to 0-1 range float from integer
#turn texture coords upside down to match spr2bmp conversion
for i in range(len(bfaceuv)):
    # add uv coordinates for untextured faces (outer airlockdoor)
    if len(bfaceuv[i]) == 0:
         if len(bfaces[i]) == 3:
             bfaceuv[i] = [0.0,0.0,0.0,0.0,0.0,0.0]
         if len(bfaces[i]) == 4:
             bfaceuv[i] = [0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]

    # add meaningfull uv coordinates for outer airlockdoor only
    if bfacemat[i] == 0x1d:
         if len(bfaces[i]) == 4:
             bfaceuv[i] = [0.0,0.0,0.0,1.0,1.0,1.0,1.0,0.0]


    #recalc existing uv coordinates
    else:
         for ii in range(len(bfaceuv[i])):
             bfaceuv[i][ii] *= 1.0
             bfaceuv[i][ii] /= 0x4000

         #turn upside down
         for ii in range(int(len(bfaceuv[i])/2)):
             bfaceuv[i][ii*2+1] -= 1
             bfaceuv[i][ii*2+1] *= -1


#built btextures list from used textures in bfacemat
# - equals whptexids number - gives ID to whptexture

iii = 0
btexids = []
btextures = []
for i in range(len(whpb2.whptexids)):
   for ii in range(len(bfacemat)):
      if whpb2.whptexids[i] == bfacemat[ii]:
         btextures.append(whpb2.whptextures[i])
         btexids.append(whpb2.whptexids[i])
         break

bfacematid = []
for i in range(len(bfacemat)):
   for ii in range(len(btexids)):
      if bfacemat[i] == btexids[ii]:
         bfacematid.append(ii)

whpb2.btexids    = btexids
whpb2.btextures  = btextures
whpb2.bvertices  = bvertices
whpb2.bfaces     = bfaces
whpb2.bfaceuv    = bfaceuv
whpb2.bfacemat   = bfacemat
whpb2.bfacematid = bfacematid


[berylraven]

Thanks a lot Duncan. Sorry I did not get back to this sooner, a lot happened in a short amount of time and I'm just now catching up on this along with some other neglected projects. I pm'ed you my email if you'd like to use that too. So anyway, hwres is missing the getshplist() method so I'm a bit stuck until I get that. I sort of get what that code will consist of, I have no idea how to go about figuring out where or how to section off the different shp's programmatically from the res file. Thanks again, not only am I getting to mess with the meshes but these scripts are a really good learning tool for me.

[duncan]

odd that the method is missing - this is what i found sofar - i must admit its been a while since writing these scripts and dont really know anymore how they work together...


#hwres.py

#Hardwar res extractor
#Duncan June 2011

#scipt does the job, but is pretty minimal
#no error checking, and assumes folders for write operations are present
#note: only extracts files of types specified in filter, and spr to subdir SPR
#doesnt create the folders if missing
#i tend to run from IDLE by copy/paste, hence the opening with 'for once...'
# to avoids indentation errors


from hwpaths import *

for once in range(1):

##read lines from res file of packed fileinfo
res = open(respath+resname,'rb')
ressize = res.seek(0,2)

import array
resheader = array.array('L')

dump = res.seek(ressize-4)
resheader.fromfile(res,1)

dump = res.seek(resheader[0])
resnumfiles = array.array('H')
resnumfiles.fromfile(res,2)

resfile = array.array('B')
resfiles = []
for i in range(resnumfiles[0]):
   resfile.fromfile(res,28)
   resfiles.append(resfile.tolist())
   for ii in range(len(resfile)):
      dump = resfile.pop()

res.close()


##decode res header filename according XOR encryption key
##also add offset and size data

global resdata
resdata = []
resfdat = array.array('L')
key  = 'SOFTWAREREFINERY'

for i in range(len(resfiles)):

   resfdat.frombytes(bytes(resfiles[i][20:28]))


   #XOR decode
   decoded = ''.join([chr(resfiles[i][4+ii]^ord(key[ii])) for ii in range(12)])

   #grab extension
   extension = decoded[9:12]

   #strip spaces
   temp =''
   for ii in range(len(decoded)):
       if decoded[ii] == ' ': pass
       else: temp = temp+decoded[ii]


   resdata.append([temp,extension,resfdat[0],resfdat[1]])
   dump = resfdat.pop()
   dump = resfdat.pop()


def findfileindex(name = ''):
for i in range(len(resdata)):
   if resdata[i][0] == name: return(i)

def getshplist():
shplist = []
for i in range(len(resdata)):
   if resdata[i][1] == 'SHP': shplist.append(resdata[i])
return(shplist)

def getfileoffset(i=-1):
      return(resdata[i][2])

def getfilesize(i=-1):
      return(resdata[i][3])



##write files, note separate function
def reswrite(dpath):

#destination path
#dpath = '/users/am/documents/projects/python/hw/hwres2blend/res/'


res = open(respath+resname,'rb')

filter = ['WLD','WXP','MAT','PAL','SPR']

for i in range(len(resdata)):
   sprfolder = ''
   for extension in filter:
      if resdata[i][0][len(resdata[i][0])-3:len(resdata[i][0])] == extension:
         if extension == 'SPR': sprfolder = 'SPR/'
         dfile = open(dpath+sprfolder+resdata[i][0],'wb')

         res.seek(resdata[i][2])
         dfile.write(res.read(resdata[i][3]))
         dfile.close()

res.close()


[berylraven]

yeah I know how that goes, I only really remember code from things I am actively programming let alone and barely remember the stuff I worked on a month ago, let alone a year so I'll take what I can get haha. Anyway It is for the most part completely functional I had to comment out the bmat.diffuse_color = ... in line 40 of ship2blend because of some array errors that I just did not feel like sorting through so I just stuck to the initial diffuse mat you provided and will use the mostly intact uv's if need be. I was really only after the meshes anyway so I consider this fantastically useful. I couldn't get the getshp() to work on it's own but I figure that is not really a problem since I got everything with getshps(). Thanks a lot and working with this reminded me of why I don't like working at the memory level and avoid it haha.

[testudines]

Is this script working for newer versions of Blender?