source: CLRX/CLRadeonExtender/trunk/CLRX/amdbin/ROCmBinaries.h @ 4947

Last change on this file since 4947 was 4947, checked in by matszpk, 13 months ago

CLRadeonExtender: ROCmBin: Write routines and classes to write MsgPack? data.

File size: 19.4 KB
Line 
1/*
2 *  CLRadeonExtender - Unofficial OpenCL Radeon Extensions Library
3 *  Copyright (C) 2014-2018 Mateusz Szpakowski
4 *
5 *  This library is free software; you can redistribute it and/or
6 *  modify it under the terms of the GNU Lesser General Public
7 *  License as published by the Free Software Foundation; either
8 *  version 2.1 of the License, or (at your option) any later version.
9 *
10 *  This library is distributed in the hope that it will be useful,
11 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 *  Lesser General Public License for more details.
14 *
15 *  You should have received a copy of the GNU Lesser General Public
16 *  License along with this library; if not, write to the Free Software
17 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
18 */
19/*! \file ROCmBinaries.h
20 * \brief ROCm binaries handling
21 */
22
23#ifndef __CLRX_ROCMBINARIES_H__
24#define __CLRX_ROCMBINARIES_H__
25
26#include <CLRX/Config.h>
27#include <cstddef>
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <vector>
32#include <CLRX/amdbin/Elf.h>
33#include <CLRX/amdbin/ElfBinaries.h>
34#include <CLRX/amdbin/Commons.h>
35#include <CLRX/utils/MemAccess.h>
36#include <CLRX/utils/Containers.h>
37#include <CLRX/utils/Utilities.h>
38#include <CLRX/utils/GPUId.h>
39#include <CLRX/utils/InputOutput.h>
40
41/// main namespace
42namespace CLRX
43{
44
45enum : Flags {
46    ROCMBIN_CREATE_REGIONMAP = 0x10,    ///< create region map
47    ROCMBIN_CREATE_METADATAINFO = 0x20,     ///< create metadata info object
48    ROCMBIN_CREATE_KERNELINFOMAP = 0x40,    ///< create kernel metadata info map
49    ROCMBIN_CREATE_ALL = ELF_CREATE_ALL | 0xfff0 ///< all ROCm binaries flags
50};
51
52/// ROCm region/symbol type
53enum class ROCmRegionType: uint8_t
54{
55    DATA,   ///< data object
56    FKERNEL,   ///< function kernel (code)
57    KERNEL  ///< OpenCL kernel to call ??
58};
59
60/// ROCm data region
61struct ROCmRegion
62{
63    CString regionName; ///< region name
64    size_t size;    ///< data size
65    size_t offset;     ///< data
66    ROCmRegionType type; ///< type
67};
68
69/// ROCm Value kind
70enum class ROCmValueKind : cxbyte
71{
72    BY_VALUE = 0,       ///< value is just value
73    GLOBAL_BUFFER,      ///< passed in global buffer
74    DYN_SHARED_PTR,     ///< passed as dynamic shared pointer
75    SAMPLER,            ///< sampler
76    IMAGE,              ///< image
77    PIPE,               ///< OpenCL pipe
78    QUEUE,              ///< OpenCL queue
79    HIDDEN_GLOBAL_OFFSET_X, ///< OpenCL global offset X
80    HIDDEN_GLOBAL_OFFSET_Y, ///< OpenCL global offset Y
81    HIDDEN_GLOBAL_OFFSET_Z, ///< OpenCL global offset Z
82    HIDDEN_NONE,            ///< none (not used)
83    HIDDEN_PRINTF_BUFFER,   ///< buffer for printf calls
84    HIDDEN_DEFAULT_QUEUE,   ///< OpenCL default queue
85    HIDDEN_COMPLETION_ACTION,    ///< ???
86    HIDDEN_MULTIGRID_SYNC_ARG, /// ???
87    MAX_VALUE = HIDDEN_MULTIGRID_SYNC_ARG
88};
89
90/// ROCm argument's value type
91enum class ROCmValueType : cxbyte
92{
93    STRUCTURE = 0,  ///< structure
94    INT8,       ///< 8-bit signed integer
95    UINT8,      ///< 8-bit unsigned integer
96    INT16,      ///< 16-bit signed integer
97    UINT16,     ///< 16-bit unsigned integer
98    FLOAT16,    ///< half floating point
99    INT32,      ///< 32-bit signed integer
100    UINT32,     ///< 32-bit unsigned integer
101    FLOAT32,    ///< single floating point
102    INT64,      ///< 64-bit signed integer
103    UINT64,     ///< 64-bit unsigned integer
104    FLOAT64,     ///< double floating point
105    MAX_VALUE = FLOAT64
106};
107
108/// ROCm argument address space
109enum class ROCmAddressSpace : cxbyte
110{
111    NONE = 0,
112    PRIVATE,
113    GLOBAL,
114    CONSTANT,
115    LOCAL,
116    GENERIC,
117    REGION,
118    MAX_VALUE = REGION
119};
120
121/// ROCm access qualifier
122enum class ROCmAccessQual: cxbyte
123{
124    DEFAULT = 0,
125    READ_ONLY,
126    WRITE_ONLY,
127    READ_WRITE,
128    MAX_VALUE = READ_WRITE
129};
130
131/// ROCm kernel argument
132struct ROCmKernelArgInfo
133{
134    CString name;       ///< name
135    CString typeName;   ///< type name
136    uint64_t size;      ///< argument size in bytes
137    union {
138        uint64_t align;     ///< argument alignment in bytes
139        uint64_t offset;
140    };
141    uint64_t pointeeAlign;      ///< alignemnt of pointed data of pointer
142    ROCmValueKind valueKind;    ///< value kind
143    ROCmValueType valueType;    ///< value type
144    ROCmAddressSpace addressSpace;  ///< pointer address space
145    ROCmAccessQual accessQual;      ///< access qualifier (for images and values)
146    ROCmAccessQual actualAccessQual;    ///< actual access qualifier
147    bool isConst;       ///< is constant
148    bool isRestrict;    ///< is restrict
149    bool isVolatile;    ///< is volatile
150    bool isPipe;        ///< is pipe
151};
152
153/// ROCm kernel metadata
154struct ROCmKernelMetadata
155{
156    CString name;       ///< kernel name
157    CString symbolName; ///< symbol name
158    std::vector<ROCmKernelArgInfo> argInfos;  ///< kernel arguments
159    CString language;       ///< language
160    cxuint langVersion[2];  ///< language version
161    cxuint reqdWorkGroupSize[3];    ///< required work group size
162    cxuint workGroupSizeHint[3];    ///< work group size hint
163    CString vecTypeHint;    ///< vector type hint
164    CString runtimeHandle;  ///< symbol of runtime handle
165    uint64_t kernargSegmentSize;    ///< kernel argument segment size
166    uint64_t groupSegmentFixedSize; ///< group segment size (fixed)
167    uint64_t privateSegmentFixedSize;   ///< private segment size (fixed)
168    uint64_t kernargSegmentAlign;       ///< alignment of kernel argument segment
169    cxuint wavefrontSize;       ///< wavefront size
170    cxuint sgprsNum;        ///< number of SGPRs
171    cxuint vgprsNum;        ///< number of VGPRs
172    uint64_t maxFlatWorkGroupSize;
173    cxuint fixedWorkGroupSize[3];
174    cxuint spilledSgprs;    ///< number of spilled SGPRs
175    cxuint spilledVgprs;    ///< number of spilled VGPRs
176    CString deviceEnqueueSymbol;
177   
178    void initialize();
179};
180
181/// ROCm printf call info
182struct ROCmPrintfInfo
183{
184    uint32_t id;    /// unique id of call
185    Array<uint32_t> argSizes;   ///< argument sizes
186    CString format;     ///< printf format
187};
188
189/// ROCm binary metadata
190struct ROCmMetadata
191{
192    cxuint version[2];  ///< version
193    std::vector<ROCmPrintfInfo> printfInfos;  ///< printf calls infos
194    std::vector<ROCmKernelMetadata> kernels;  ///< kernel metadatas
195   
196    /// initialize metadata info
197    void initialize();
198    /// parse metadata info from metadata string
199    void parse(size_t metadataSize, const char* metadata);
200    /// parse metadata info from MsgPack
201    void parseMsgPack(size_t metadataSize, const cxbyte* metadata);
202};
203
204struct ROCmKernelDescriptor
205{
206    uint32_t groupSegmentFixedSize;
207    uint32_t privateSegmentFixedSize;
208    uint64_t reserved0;
209    uint64_t kernelCodeEntryOffset;
210    uint64_t reserved1;
211    cxbyte reserved2[12];
212    uint32_t pgmRsrc3;
213    uint32_t pgmRsrc1;
214    uint32_t pgmRsrc2;
215    uint16_t initialKernelExecState;
216    cxbyte reserved3[6];
217   
218    void toLE()
219    {
220        SLEV(groupSegmentFixedSize, groupSegmentFixedSize);
221        SLEV(privateSegmentFixedSize, privateSegmentFixedSize);
222        SLEV(kernelCodeEntryOffset, kernelCodeEntryOffset);
223        SLEV(pgmRsrc3, pgmRsrc3);
224        SLEV(pgmRsrc1, pgmRsrc1);
225        SLEV(pgmRsrc2, pgmRsrc2);
226        SLEV(initialKernelExecState, initialKernelExecState);
227    }
228};
229
230/// ROCm main binary for GPU for 64-bit mode
231/** This object doesn't copy binary code content.
232 * Only it takes and uses a binary code.
233 */
234class ROCmBinary : public ElfBinary64, public NonCopyableAndNonMovable
235{
236public:
237    /// region map type
238    typedef Array<std::pair<CString, size_t> > RegionMap;
239private:
240    size_t regionsNum;
241    std::unique_ptr<ROCmRegion[]> regions;  ///< AMD metadatas
242    RegionMap regionsMap;
243    size_t codeSize;
244    cxbyte* code;
245    size_t globalDataSize;
246    cxbyte* globalData;
247    CString target;
248    size_t metadataSize;
249    char* metadata;
250    std::unique_ptr<ROCmMetadata> metadataInfo;
251    RegionMap kernelInfosMap;
252    Array<const ROCmKernelDescriptor*> kernelDescs;
253    Array<size_t> gotSymbols;
254    bool newBinFormat;
255    bool llvm10BinFormat;
256    bool metadataV3Format;
257public:
258    /// constructor
259    ROCmBinary(size_t binaryCodeSize, cxbyte* binaryCode,
260            Flags creationFlags = ROCMBIN_CREATE_ALL);
261    /// default destructor
262    ~ROCmBinary() = default;
263   
264    /// determine GPU device type from this binary
265    GPUDeviceType determineGPUDeviceType(uint32_t& archMinor,
266                     uint32_t& archStepping) const;
267   
268    /// get regions number
269    size_t getRegionsNum() const
270    { return regionsNum; }
271   
272    /// get region by index
273    const ROCmRegion& getRegion(size_t index) const
274    { return regions[index]; }
275   
276    /// get region by name
277    const ROCmRegion& getRegion(const char* name) const;
278   
279    /// get code size
280    size_t getCodeSize() const
281    { return codeSize; }
282    /// get code
283    const cxbyte* getCode() const
284    { return code; }
285    /// get code
286    cxbyte* getCode()
287    { return code; }
288   
289    /// get global data size
290    size_t getGlobalDataSize() const
291    { return globalDataSize; }
292   
293    /// get global data
294    const cxbyte* getGlobalData() const
295    { return globalData; }
296    /// get global data
297    cxbyte* getGlobalData()
298    { return globalData; }
299   
300    /// get metadata size
301    size_t getMetadataSize() const
302    { return metadataSize; }
303    /// get metadata
304    const char* getMetadata() const
305    { return metadata; }
306    /// get metadata
307    char* getMetadata()
308    { return metadata; }
309   
310    /// has metadata info
311    bool hasMetadataInfo() const
312    { return metadataInfo!=nullptr; }
313   
314    /// get metadata info
315    const ROCmMetadata& getMetadataInfo() const
316    { return *metadataInfo; }
317   
318    /// get kernel metadata infos number
319    size_t getKernelInfosNum() const
320    { return metadataInfo->kernels.size(); }
321   
322    /// get kernel metadata info
323    const ROCmKernelMetadata& getKernelInfo(size_t index) const
324    { return metadataInfo->kernels[index]; }
325   
326    /// get kernel metadata info by name
327    const ROCmKernelMetadata& getKernelInfo(const char* name) const;
328   
329    /// get kernel descriptor
330    const ROCmKernelDescriptor* getKernelDescriptor(size_t index) const
331    { return kernelDescs[index]; }
332    // get kernel descriptor by name
333    const ROCmKernelDescriptor* getKernelDescriptor(const char* name) const;
334   
335    /// get target
336    const CString& getTarget() const
337    { return target; }
338   
339    /// return true is new binary format
340    bool isNewBinaryFormat() const
341    { return newBinFormat; }
342   
343    /// return true is LLVM10 binary format
344    bool isLLVM10BinaryFormat() const
345    { return llvm10BinFormat; }
346   
347    /// return true is metadata V3 code object format
348    bool isMetadataV3Format() const
349    { return metadataV3Format; }
350   
351    /// get GOT symbol index (from elfbin dynsymbols)
352    size_t getGotSymbolsNum() const
353    { return gotSymbols.size(); }
354   
355    /// get GOT symbols (indices) (from elfbin dynsymbols)
356    const Array<size_t> getGotSymbols() const
357    { return gotSymbols; }
358   
359    /// get GOT symbol index (from elfbin dynsymbols)
360    size_t getGotSymbol(size_t index) const
361    { return gotSymbols[index]; }
362   
363    /// returns true if kernel map exists
364    bool hasRegionMap() const
365    { return (creationFlags & ROCMBIN_CREATE_REGIONMAP) != 0; }
366    /// returns true if object has kernel map
367    bool hasKernelInfoMap() const
368    { return (creationFlags & ROCMBIN_CREATE_KERNELINFOMAP) != 0; }
369};
370
371enum {
372    ROCMFLAG_USE_PRIVATE_SEGMENT_BUFFER = AMDHSAFLAG_USE_PRIVATE_SEGMENT_BUFFER,
373    ROCMFLAG_USE_DISPATCH_PTR = AMDHSAFLAG_USE_DISPATCH_PTR,
374    ROCMFLAG_USE_QUEUE_PTR = AMDHSAFLAG_USE_QUEUE_PTR,
375    ROCMFLAG_USE_KERNARG_SEGMENT_PTR = AMDHSAFLAG_USE_KERNARG_SEGMENT_PTR,
376    ROCMFLAG_USE_DISPATCH_ID = AMDHSAFLAG_USE_DISPATCH_ID,
377    ROCMFLAG_USE_FLAT_SCRATCH_INIT = AMDHSAFLAG_USE_FLAT_SCRATCH_INIT,
378    ROCMFLAG_USE_PRIVATE_SEGMENT_SIZE = AMDHSAFLAG_USE_PRIVATE_SEGMENT_SIZE,
379    ROCMFLAG_USE_GRID_WORKGROUP_COUNT_BIT = AMDHSAFLAG_USE_GRID_WORKGROUP_COUNT_BIT,
380    ROCMFLAG_USE_GRID_WORKGROUP_COUNT_X = AMDHSAFLAG_USE_GRID_WORKGROUP_COUNT_X,
381    ROCMFLAG_USE_GRID_WORKGROUP_COUNT_Y = AMDHSAFLAG_USE_GRID_WORKGROUP_COUNT_Y,
382    ROCMFLAG_USE_GRID_WORKGROUP_COUNT_Z = AMDHSAFLAG_USE_GRID_WORKGROUP_COUNT_Z,
383    ROCMFLAG_USE_WAVE32 = AMDHSAFLAG_USE_WAVE32,
384   
385    ROCMFLAG_USE_ORDERED_APPEND_GDS = AMDHSAFLAG_USE_ORDERED_APPEND_GDS,
386    ROCMFLAG_PRIVATE_ELEM_SIZE_BIT = AMDHSAFLAG_PRIVATE_ELEM_SIZE_BIT,
387    ROCMFLAG_USE_PTR64 = AMDHSAFLAG_USE_PTR64,
388    ROCMFLAG_USE_DYNAMIC_CALL_STACK = AMDHSAFLAG_USE_DYNAMIC_CALL_STACK,
389    ROCMFLAG_USE_DEBUG_ENABLED = AMDHSAFLAG_USE_DEBUG_ENABLED,
390    ROCMFLAG_USE_XNACK_ENABLED = AMDHSAFLAG_USE_XNACK_ENABLED
391};
392
393/// ROCm kernel configuration structure
394typedef AmdHsaKernelConfig ROCmKernelConfig;
395
396/// check whether is Amd OpenCL 2.0 binary
397extern bool isROCmBinary(size_t binarySize, const cxbyte* binary);
398
399/*
400 * ROCm Binary Generator
401 */
402
403enum: cxuint {
404    ROCMSECTID_HASH = ELFSECTID_OTHER_BUILTIN,
405    ROCMSECTID_DYNAMIC,
406    ROCMSECTID_NOTE,
407    ROCMSECTID_GPUCONFIG,
408    ROCMSECTID_RELADYN,
409    ROCMSECTID_GOT,
410    ROCMSECTID_MAX = ROCMSECTID_GOT
411};
412
413/// ROCm binary symbol input
414struct ROCmSymbolInput
415{
416    CString symbolName; ///< symbol name
417    size_t offset;  ///< offset in code
418    size_t size;    ///< size of symbol
419    ROCmRegionType type;  ///< type
420};
421
422/// ROCm binary input structure
423struct ROCmInput
424{
425    GPUDeviceType deviceType;   ///< GPU device type
426    uint32_t archMinor;         ///< GPU arch minor
427    uint32_t archStepping;      ///< GPU arch stepping
428    uint32_t eflags;    ///< ELF headef e_flags field
429    bool newBinFormat;       ///< use new binary format for ROCm
430    bool llvm10BinFormat;
431    bool metadataV3Format;
432    size_t globalDataSize;  ///< global data size
433    const cxbyte* globalData;   ///< global data
434    std::vector<ROCmSymbolInput> symbols;   ///< symbols
435    size_t codeSize;        ///< code size
436    const cxbyte* code;     ///< code
437    size_t commentSize; ///< comment size (can be null)
438    const char* comment; ///< comment
439    CString target;     ///< LLVM target triple with device name
440    CString targetTripple; ///< same LLVM target tripple
441    size_t metadataSize;    ///< metadata size
442    const char* metadata;   ///< metadata
443    bool useMetadataInfo;   ///< use metadatainfo instead same metadata
444    ROCmMetadata metadataInfo; ///< metadata info
445   
446    /// list of indices of symbols to GOT section
447    /**  list of indices of symbols to GOT section.
448     * If symbol index is lower than symbols.size(), then it refer to symbols
449     * otherwise it refer to extraSymbols (index - symbols.size())
450     */
451    std::vector<size_t> gotSymbols;
452    std::vector<BinSection> extraSections;  ///< extra sections
453    std::vector<BinSymbol> extraSymbols;    ///< extra symbols
454   
455    /// add empty kernel with default values
456    void addEmptyKernel(const char* kernelName);
457};
458
459/// ROCm binary generator
460class ROCmBinGenerator: public NonCopyableAndNonMovable
461{
462private:
463    private:
464    bool manageable;
465    const ROCmInput* input;
466    std::unique_ptr<ElfBinaryGen64> elfBinGen64;
467    size_t binarySize;
468    size_t commentSize;
469    const char* comment;
470    std::string target;
471    std::unique_ptr<cxbyte[]> noteBuf;
472    std::string metadataStr;
473    size_t metadataSize;
474    const char* metadata;
475    cxuint mainSectionsNum;
476    uint16_t mainBuiltinSectTable[ROCMSECTID_MAX-ELFSECTID_START+1];
477    void* rocmGotGen;
478    void* rocmRelaDynGen;
479   
480    void generateInternal(std::ostream* osPtr, std::vector<char>* vPtr,
481             Array<cxbyte>* aPtr);
482public:
483    /// constructor
484    ROCmBinGenerator();
485    /// constructor with ROCm input
486    explicit ROCmBinGenerator(const ROCmInput* rocmInput);
487   
488    /// constructor
489    /**
490     * \param deviceType device type
491     * \param archMinor architecture minor number
492     * \param archStepping architecture stepping number
493     * \param codeSize size of code
494     * \param code code pointer
495     * \param globalDataSize size of global data
496     * \param globalData global data pointer
497     * \param symbols symbols (kernels, datas,...)
498     */
499    ROCmBinGenerator(GPUDeviceType deviceType, uint32_t archMinor, uint32_t archStepping,
500            size_t codeSize, const cxbyte* code,
501            size_t globalDataSize, const cxbyte* globalData,
502            const std::vector<ROCmSymbolInput>& symbols);
503    /// constructor
504    ROCmBinGenerator(GPUDeviceType deviceType, uint32_t archMinor, uint32_t archStepping,
505            size_t codeSize, const cxbyte* code,
506            size_t globalDataSize, const cxbyte* globalData,
507            std::vector<ROCmSymbolInput>&& symbols);
508    /// destructor
509    ~ROCmBinGenerator();
510   
511    /// get input
512    const ROCmInput* getInput() const
513    { return input; }
514    /// set input
515    void setInput(const ROCmInput* input);
516   
517    /// prepare binary generator (for section diffs)
518    void prepareBinaryGen();
519    /// get section offset (from main section)
520    size_t getSectionOffset(cxuint sectionId) const
521    { return elfBinGen64->getRegionOffset(
522                    mainBuiltinSectTable[sectionId - ELFSECTID_START]); }
523    /// update symbols
524    void updateSymbols();
525   
526    /// generates binary to array of bytes
527    void generate(Array<cxbyte>& array);
528   
529    /// generates binary to output stream
530    void generate(std::ostream& os);
531   
532    /// generates binary to vector of char
533    void generate(std::vector<char>& vector);
534};
535
536void generateROCmMetadata(const ROCmMetadata& mdInfo,
537                    const ROCmKernelConfig** kconfigs, std::string& output);
538
539void generateROCmMetadataMsgPack(const ROCmMetadata& mdInfo,
540                    const ROCmKernelConfig** kconfigs, std::vector<cxbyte>& output);
541
542void parseROCmMetadata(size_t metadataSize, const char* metadata,
543                ROCmMetadata& metadataInfo);
544
545void parseROCmMetadataMsgPack(size_t metadataSize, const cxbyte* metadata,
546                ROCmMetadata& metadataInfo);
547
548class MsgPackMapParser;
549
550class MsgPackArrayParser
551{
552private:
553    const cxbyte*& dataPtr;
554    const cxbyte* dataEnd;
555    size_t count;
556    void handleErrors();
557public:
558    MsgPackArrayParser(const cxbyte*& _dataPtr, const cxbyte* _dataEnd);
559   
560    void parseNil();
561    bool parseBool();
562    uint64_t parseInteger(cxbyte signess);
563    double parseFloat();
564    std::string parseString();
565    Array<cxbyte> parseData();
566    MsgPackArrayParser parseArray();
567    MsgPackMapParser parseMap();
568    size_t end(); // return left elements
569   
570    bool haveElements() const
571    { return count!=0; }
572};
573
574enum: cxbyte {
575    MSGPACK_WS_UNSIGNED = 0,  // only unsigned
576    MSGPACK_WS_SIGNED = 1,  // only signed
577    MSGPACK_WS_BOTH = 2  // both signed and unsigned range checking
578};
579
580class MsgPackMapParser
581{
582private:
583    const cxbyte*& dataPtr;
584    const cxbyte* dataEnd;
585    size_t count;
586    bool keyLeft;
587    void handleErrors(bool key);
588public:
589    MsgPackMapParser(const cxbyte*& _dataPtr, const cxbyte* _dataEnd);
590   
591    void parseKeyNil();
592    bool parseKeyBool();
593    uint64_t parseKeyInteger(cxbyte signess);
594    double parseKeyFloat();
595    std::string parseKeyString();
596    Array<cxbyte> parseKeyData();
597    MsgPackArrayParser parseKeyArray();
598    MsgPackMapParser parseKeyMap();
599    void parseValueNil();
600    bool parseValueBool();
601    uint64_t parseValueInteger(cxbyte signess);
602    double parseValueFloat();
603    std::string parseValueString();
604    Array<cxbyte> parseValueData();
605    MsgPackArrayParser parseValueArray();
606    MsgPackMapParser parseValueMap();
607    void skipValue();
608    size_t end(); // return left elements
609   
610    bool haveElements() const
611    { return count!=0; }
612};
613
614};
615
616#endif
Note: See TracBrowser for help on using the repository browser.