1 /* Copyright (c) 2012-2019 The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
14 #include "kgsl_sharedmem.h"
15 #include "kgsl_snapshot.h"
18 #include "adreno_pm4types.h"
20 #include "adreno_cp_parser.h"
21 #include "adreno_snapshot.h"
22 #include "adreno_a5xx.h"
24 #define VPC_MEMORY_BANKS 4
26 /* Maintain a list of the objects we see during parsing */
28 #define SNAPSHOT_OBJ_BUFSIZE 64
30 /* Used to print error message if an IB has too many objects in it */
31 static int ib_max_objs;
33 struct snapshot_rb_params {
34 struct kgsl_snapshot *snapshot;
35 struct adreno_ringbuffer *rb;
38 /* Keep track of how many bytes are frozen after a snapshot and tell the user */
39 static size_t snapshot_frozen_objsize;
41 static struct kgsl_snapshot_object objbuf[SNAPSHOT_OBJ_BUFSIZE];
43 /* Pointer to the next open entry in the object list */
44 static unsigned int objbufptr;
46 static inline int adreno_rb_ctxtswitch(struct adreno_device *adreno_dev,
49 return cmd[0] == cp_packet(adreno_dev, CP_NOP, 1) &&
50 cmd[1] == KGSL_CONTEXT_TO_MEM_IDENTIFIER;
53 /* Push a new buffer object onto the list */
54 void kgsl_snapshot_push_object(struct kgsl_process_private *process,
55 uint64_t gpuaddr, uint64_t dwords)
58 struct kgsl_mem_entry *entry;
64 * Sometimes IBs can be reused in the same dump. Because we parse from
65 * oldest to newest, if we come across an IB that has already been used,
66 * assume that it has been reused and update the list with the newest
70 for (index = 0; index < objbufptr; index++) {
71 if (objbuf[index].gpuaddr == gpuaddr &&
72 objbuf[index].entry->priv == process) {
74 * Check if newly requested size is within the
75 * allocated range or not, otherwise continue
78 if (!kgsl_gpuaddr_in_memdesc(
79 &objbuf[index].entry->memdesc,
80 gpuaddr, dwords << 2)) {
82 "snapshot: IB 0x%016llx size is not within the memdesc range\n",
87 objbuf[index].size = max_t(uint64_t,
94 if (objbufptr == SNAPSHOT_OBJ_BUFSIZE) {
95 KGSL_CORE_ERR("snapshot: too many snapshot objects\n");
99 entry = kgsl_sharedmem_find(process, gpuaddr);
101 KGSL_CORE_ERR("snapshot: Can't find entry for 0x%016llX\n",
106 if (!kgsl_gpuaddr_in_memdesc(&entry->memdesc, gpuaddr, dwords << 2)) {
107 KGSL_CORE_ERR("snapshot: Mem entry 0x%016llX is too small\n",
109 kgsl_mem_entry_put(entry);
113 /* Put it on the list of things to parse */
114 objbuf[objbufptr].gpuaddr = gpuaddr;
115 objbuf[objbufptr].size = dwords << 2;
116 objbuf[objbufptr++].entry = entry;
120 * Returns index of the specified object is already on the list of buffers
124 static int find_object(uint64_t gpuaddr, struct kgsl_process_private *process)
128 for (index = 0; index < objbufptr; index++) {
129 if (objbuf[index].gpuaddr == gpuaddr &&
130 objbuf[index].entry->priv == process)
137 * snapshot_freeze_obj_list() - Take a list of ib objects and freeze their
138 * memory for snapshot
139 * @snapshot: The snapshot data.
140 * @process: The process to which the IB belongs
141 * @ib_obj_list: List of the IB objects
143 * Returns 0 on success else error code
145 static int snapshot_freeze_obj_list(struct kgsl_snapshot *snapshot,
146 struct kgsl_process_private *process,
147 struct adreno_ib_object_list *ib_obj_list)
150 struct adreno_ib_object *ib_objs;
153 for (i = 0; i < ib_obj_list->num_objs; i++) {
158 ib_objs = &(ib_obj_list->obj_list[i]);
159 /* Make sure this object is not going to be saved statically */
160 for (index = 0; index < objbufptr; index++) {
161 if ((objbuf[index].gpuaddr <= ib_objs->gpuaddr) &&
162 ((objbuf[index].gpuaddr +
163 (objbuf[index].size)) >=
164 (ib_objs->gpuaddr + ib_objs->size)) &&
165 (objbuf[index].entry->priv == process)) {
172 temp_ret = kgsl_snapshot_get_object(snapshot,
173 process, ib_objs->gpuaddr,
175 ib_objs->snapshot_obj_type);
180 snapshot_frozen_objsize += temp_ret;
188 * We want to store the last executed IB1 and IB2 in the static region to ensure
189 * that we get at least some information out of the snapshot even if we can't
190 * access the dynamic data from the sysfs file. Push all other IBs on the
193 static inline void parse_ib(struct kgsl_device *device,
194 struct kgsl_snapshot *snapshot,
195 struct kgsl_process_private *process,
196 uint64_t gpuaddr, uint64_t dwords)
198 struct adreno_ib_object_list *ib_obj_list;
201 * Check the IB address - if it is either the last executed IB1
202 * then push it into the static blob otherwise put it in the dynamic
205 if (gpuaddr == snapshot->ib1base) {
206 kgsl_snapshot_push_object(process, gpuaddr, dwords);
210 if (kgsl_snapshot_have_object(snapshot, process,
211 gpuaddr, dwords << 2))
214 if (-E2BIG == adreno_ib_create_object_list(device, process,
215 gpuaddr, dwords, snapshot->ib2base,
220 kgsl_snapshot_add_ib_obj_list(snapshot, ib_obj_list);
224 static inline bool iommu_is_setstate_addr(struct kgsl_device *device,
225 uint64_t gpuaddr, uint64_t size)
227 struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
229 if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_IOMMU)
232 return kgsl_gpuaddr_in_memdesc(&iommu->setstate, gpuaddr,
236 static void dump_all_ibs(struct kgsl_device *device,
237 struct adreno_ringbuffer *rb,
238 struct kgsl_snapshot *snapshot)
242 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
244 rbptr = rb->buffer_desc.hostptr;
246 for (index = 0; index < KGSL_RB_DWORDS;) {
248 if (adreno_cmd_is_ib(adreno_dev, rbptr[index])) {
252 if (ADRENO_LEGACY_PM4(adreno_dev)) {
253 ibaddr = rbptr[index + 1];
254 ibsize = rbptr[index + 2];
257 ibaddr = rbptr[index + 2];
258 ibaddr = ibaddr << 32 | rbptr[index + 1];
259 ibsize = rbptr[index + 3];
263 /* Don't parse known global IBs */
264 if (iommu_is_setstate_addr(device, ibaddr, ibsize))
267 if (kgsl_gpuaddr_in_memdesc(&adreno_dev->pwron_fixup,
271 parse_ib(device, snapshot, snapshot->process, ibaddr,
279 * snapshot_rb_ibs() - Dump rb data and capture the IB's in the RB as well
280 * @device: Pointer to a KGSL device
281 * @rb: The RB to dump
282 * @data: Pointer to memory where the RB data is to be dumped
283 * @snapshot: Pointer to information about the current snapshot being taken
285 static void snapshot_rb_ibs(struct kgsl_device *device,
286 struct adreno_ringbuffer *rb,
287 struct kgsl_snapshot *snapshot)
289 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
290 unsigned int rptr, *rbptr;
292 int parse_ibs = 0, ib_parse_start;
294 /* Get the current read pointers for the RB */
295 adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &rptr);
298 * Figure out the window of ringbuffer data to dump. First we need to
299 * find where the last processed IB ws submitted. Start walking back
304 rbptr = rb->buffer_desc.hostptr;
310 if (ADRENO_LEGACY_PM4(adreno_dev))
311 index = KGSL_RB_DWORDS - 3;
313 index = KGSL_RB_DWORDS - 4;
315 /* We wrapped without finding what we wanted */
316 if (index < rb->wptr) {
322 if (adreno_cmd_is_ib(adreno_dev, rbptr[index])) {
323 if (ADRENO_LEGACY_PM4(adreno_dev)) {
324 if (rbptr[index + 1] == snapshot->ib1base)
329 ibaddr = rbptr[index + 2];
330 ibaddr = ibaddr << 32 | rbptr[index + 1];
331 if (ibaddr == snapshot->ib1base)
335 } while (index != rb->wptr);
338 * If the ib1 was not found, for example, if ib1base was restored
339 * incorrectly after preemption, then simply dump the entire
340 * ringbuffer along with all the IBs in the ringbuffer.
343 if (index == rb->wptr) {
344 dump_all_ibs(device, rb, snapshot);
349 * index points at the last submitted IB. We can only trust that the
350 * memory between the context switch and the hanging IB is valid, so
351 * the next step is to find the context switch before the submission
354 while (index != rb->wptr) {
358 index = KGSL_RB_DWORDS - 2;
361 * Wrapped without finding the context switch. This is
362 * harmless - we should still have enough data to dump a
366 if (index < rb->wptr) {
372 /* Break if the current packet is a context switch identifier */
373 if ((rbptr[index] == cp_packet(adreno_dev, CP_NOP, 1)) &&
374 (rbptr[index + 1] == KGSL_CONTEXT_TO_MEM_IDENTIFIER))
379 * Index represents the start of the window of interest. We will try
380 * to dump all buffers between here and the rptr
383 ib_parse_start = index;
386 * Loop through the RB, looking for indirect buffers and MMU pagetable
391 for (i = 0; i < KGSL_RB_DWORDS; i++) {
393 * Only parse IBs between the start and the rptr or the next
394 * context switch, whichever comes first
397 if (parse_ibs == 0 && index == ib_parse_start)
399 else if (index == rptr || adreno_rb_ctxtswitch(adreno_dev,
403 if (parse_ibs && adreno_cmd_is_ib(adreno_dev, rbptr[index])) {
407 if (ADRENO_LEGACY_PM4(adreno_dev)) {
408 ibaddr = rbptr[index + 1];
409 ibsize = rbptr[index + 2];
411 ibaddr = rbptr[index + 2];
412 ibaddr = ibaddr << 32 | rbptr[index + 1];
413 ibsize = rbptr[index + 3];
416 index = (index + 1) % KGSL_RB_DWORDS;
418 /* Don't parse known global IBs */
419 if (iommu_is_setstate_addr(device, ibaddr, ibsize))
422 if (kgsl_gpuaddr_in_memdesc(&adreno_dev->pwron_fixup,
426 parse_ib(device, snapshot, snapshot->process,
429 index = (index + 1) % KGSL_RB_DWORDS;
434 /* Snapshot the ringbuffer memory */
435 static size_t snapshot_rb(struct kgsl_device *device, u8 *buf,
436 size_t remain, void *priv)
438 struct kgsl_snapshot_rb_v2 *header = (struct kgsl_snapshot_rb_v2 *)buf;
439 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
440 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
441 struct snapshot_rb_params *snap_rb_params = priv;
442 struct kgsl_snapshot *snapshot = snap_rb_params->snapshot;
443 struct adreno_ringbuffer *rb = snap_rb_params->rb;
446 * Dump the entire ringbuffer - the parser can choose how much of it to
450 if (remain < KGSL_RB_SIZE + sizeof(*header)) {
451 KGSL_CORE_ERR("snapshot: Not enough memory for the rb section");
455 /* Write the sub-header for the section */
457 header->end = KGSL_RB_DWORDS;
458 header->wptr = rb->wptr;
459 header->rptr = adreno_get_rptr(rb);
460 header->rbsize = KGSL_RB_DWORDS;
461 header->count = KGSL_RB_DWORDS;
462 adreno_rb_readtimestamp(adreno_dev, rb, KGSL_TIMESTAMP_QUEUED,
463 &header->timestamp_queued);
464 adreno_rb_readtimestamp(adreno_dev, rb, KGSL_TIMESTAMP_RETIRED,
465 &header->timestamp_retired);
466 header->gpuaddr = rb->buffer_desc.gpuaddr;
469 if (rb == adreno_dev->cur_rb)
470 snapshot_rb_ibs(device, rb, snapshot);
472 /* Just copy the ringbuffer, there are no active IBs */
473 memcpy(data, rb->buffer_desc.hostptr, KGSL_RB_SIZE);
475 /* Return the size of the section */
476 return KGSL_RB_SIZE + sizeof(*header);
479 static int _count_mem_entries(int id, void *ptr, void *data)
492 static int _save_mem_entries(int id, void *ptr, void *data)
494 struct kgsl_mem_entry *entry = ptr;
495 struct mem_entry *m = (struct mem_entry *) data;
496 unsigned int index = id - 1;
498 m[index].gpuaddr = entry->memdesc.gpuaddr;
499 m[index].size = entry->memdesc.size;
500 m[index].type = kgsl_memdesc_get_memtype(&entry->memdesc);
505 static size_t snapshot_capture_mem_list(struct kgsl_device *device,
506 u8 *buf, size_t remain, void *priv)
508 struct kgsl_snapshot_mem_list_v2 *header =
509 (struct kgsl_snapshot_mem_list_v2 *)buf;
512 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
513 struct kgsl_process_private *process = priv;
515 /* we need a process to search! */
519 spin_lock(&process->mem_lock);
521 /* We need to know the number of memory objects that the process has */
522 idr_for_each(&process->mem_idr, _count_mem_entries, &num_mem);
527 if (remain < ((num_mem * sizeof(struct mem_entry)) + sizeof(*header))) {
528 KGSL_CORE_ERR("snapshot: Not enough memory for the mem list");
532 header->num_entries = num_mem;
533 header->ptbase = kgsl_mmu_pagetable_get_ttbr0(process->pagetable);
535 * Walk throught the memory list and store the
536 * tuples(gpuaddr, size, memtype) in snapshot
539 idr_for_each(&process->mem_idr, _save_mem_entries, data);
541 ret = sizeof(*header) + (num_mem * sizeof(struct mem_entry));
543 spin_unlock(&process->mem_lock);
547 struct snapshot_ib_meta {
548 struct kgsl_snapshot *snapshot;
549 struct kgsl_snapshot_object *obj;
556 void kgsl_snapshot_add_active_ib_obj_list(struct kgsl_device *device,
557 struct kgsl_snapshot *snapshot)
559 struct adreno_ib_object_list *ib_obj_list;
562 if (!snapshot->ib1dumped)
563 index = find_object(snapshot->ib1base, snapshot->process);
565 /* only do this for IB1 because the IB2's are part of IB1 objects */
566 if ((index != -ENOENT) &&
567 (snapshot->ib1base == objbuf[index].gpuaddr)) {
568 if (-E2BIG == adreno_ib_create_object_list(device,
569 objbuf[index].entry->priv,
570 objbuf[index].gpuaddr,
571 objbuf[index].size >> 2,
576 /* freeze the IB objects in the IB */
577 snapshot_freeze_obj_list(snapshot,
578 objbuf[index].entry->priv,
580 adreno_ib_destroy_obj_list(ib_obj_list);
583 /* Get the IB2 index from parsed object */
584 index = find_object(snapshot->ib2base, snapshot->process);
586 if (index != -ENOENT)
587 parse_ib(device, snapshot, snapshot->process,
588 snapshot->ib2base, objbuf[index].size >> 2);
593 * active_ib_is_parsed() - Checks if active ib is already parsed
594 * @gpuaddr: Active IB base address at the time of fault
595 * @size: Active IB size
596 * @process: The process to which the IB belongs
598 * Function returns true if the active is already is parsed
601 static bool active_ib_is_parsed(uint64_t gpuaddr, uint64_t size,
602 struct kgsl_process_private *process)
605 /* go through the static list for gpuaddr is in list or not */
606 for (index = 0; index < objbufptr; index++) {
607 if ((objbuf[index].gpuaddr <= gpuaddr) &&
608 ((objbuf[index].gpuaddr +
609 (objbuf[index].size)) >=
611 (objbuf[index].entry->priv == process))
616 /* Snapshot the memory for an indirect buffer */
617 static size_t snapshot_ib(struct kgsl_device *device, u8 *buf,
618 size_t remain, void *priv)
620 struct kgsl_snapshot_ib_v2 *header = (struct kgsl_snapshot_ib_v2 *)buf;
621 struct snapshot_ib_meta *meta = priv;
623 unsigned int *dst = (unsigned int *)(buf + sizeof(*header));
624 struct adreno_ib_object_list *ib_obj_list;
625 struct kgsl_snapshot *snapshot;
626 struct kgsl_snapshot_object *obj;
627 struct kgsl_memdesc *memdesc;
629 if (meta == NULL || meta->snapshot == NULL || meta->obj == NULL) {
630 KGSL_CORE_ERR("snapshot: bad metadata");
633 snapshot = meta->snapshot;
635 memdesc = &obj->entry->memdesc;
637 /* If size is zero get it from the medesc size */
639 obj->size = (memdesc->size - (obj->gpuaddr - memdesc->gpuaddr));
641 if (remain < (obj->size + sizeof(*header))) {
642 KGSL_CORE_ERR("snapshot: Not enough memory for the ib\n");
646 src = kgsl_gpuaddr_to_vaddr(memdesc, obj->gpuaddr);
649 "snapshot: Unable to map GPU memory object 0x%016llX into the kernel\n",
654 /* only do this for IB1 because the IB2's are part of IB1 objects */
655 if (meta->ib1base == obj->gpuaddr) {
657 snapshot->ib1dumped = active_ib_is_parsed(obj->gpuaddr,
658 obj->size, obj->entry->priv);
659 if (-E2BIG == adreno_ib_create_object_list(device,
661 obj->gpuaddr, obj->size >> 2,
666 /* freeze the IB objects in the IB */
667 snapshot_freeze_obj_list(snapshot,
670 adreno_ib_destroy_obj_list(ib_obj_list);
675 if (meta->ib2base == obj->gpuaddr)
676 snapshot->ib2dumped = active_ib_is_parsed(obj->gpuaddr,
677 obj->size, obj->entry->priv);
679 /* Write the sub-header for the section */
680 header->gpuaddr = obj->gpuaddr;
682 kgsl_mmu_pagetable_get_ttbr0(obj->entry->priv->pagetable);
683 header->size = obj->size >> 2;
685 /* Write the contents of the ib */
686 memcpy((void *)dst, (void *)src, (size_t) obj->size);
687 /* Write the contents of the ib */
689 return obj->size + sizeof(*header);
692 /* Dump another item on the current pending list */
693 static void dump_object(struct kgsl_device *device, int obj,
694 struct kgsl_snapshot *snapshot)
696 struct snapshot_ib_meta meta;
698 meta.snapshot = snapshot;
699 meta.obj = &objbuf[obj];
700 meta.ib1base = snapshot->ib1base;
701 meta.ib1size = snapshot->ib1size;
702 meta.ib2base = snapshot->ib2base;
703 meta.ib2size = snapshot->ib2size;
705 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_IB_V2,
706 snapshot, snapshot_ib, &meta);
707 if (objbuf[obj].entry) {
708 kgsl_memdesc_unmap(&(objbuf[obj].entry->memdesc));
709 kgsl_mem_entry_put(objbuf[obj].entry);
713 /* setup_fault process - Find kgsl_process_private struct that caused the fault
715 * Find the faulting process based what the dispatcher thinks happened and
716 * what the hardware is using for the current pagetable. The process struct
717 * will be used to look up GPU addresses that are encountered while parsing
720 static void setup_fault_process(struct kgsl_device *device,
721 struct kgsl_snapshot *snapshot,
722 struct kgsl_process_private *process)
724 u64 hw_ptbase, proc_ptbase;
726 if (process != NULL && !kgsl_process_private_get(process))
729 /* Get the physical address of the MMU pagetable */
730 hw_ptbase = kgsl_mmu_get_current_ttbr0(&device->mmu);
732 /* if we have an input process, make sure the ptbases match */
734 proc_ptbase = kgsl_mmu_pagetable_get_ttbr0(process->pagetable);
735 /* agreement! No need to check further */
736 if (hw_ptbase == proc_ptbase)
739 kgsl_process_private_put(process);
741 KGSL_CORE_ERR("snapshot: ptbase mismatch hw %llx sw %llx\n",
742 hw_ptbase, proc_ptbase);
745 /* try to find the right pagetable by walking the process list */
746 if (kgsl_mmu_is_perprocess(&device->mmu)) {
747 struct kgsl_process_private *tmp;
749 mutex_lock(&kgsl_driver.process_mutex);
750 list_for_each_entry(tmp, &kgsl_driver.process_list, list) {
753 pt_ttbr0 = kgsl_mmu_pagetable_get_ttbr0(tmp->pagetable);
754 if ((pt_ttbr0 == hw_ptbase)
755 && kgsl_process_private_get(tmp)) {
760 mutex_unlock(&kgsl_driver.process_mutex);
763 snapshot->process = process;
766 /* Snapshot a global memory buffer */
767 static size_t snapshot_global(struct kgsl_device *device, u8 *buf,
768 size_t remain, void *priv)
770 struct kgsl_memdesc *memdesc = priv;
772 struct kgsl_snapshot_gpu_object_v2 *header =
773 (struct kgsl_snapshot_gpu_object_v2 *)buf;
775 u8 *ptr = buf + sizeof(*header);
777 if (memdesc->size == 0)
780 if (remain < (memdesc->size + sizeof(*header))) {
781 KGSL_CORE_ERR("snapshot: Not enough memory for the memdesc\n");
785 if (memdesc->hostptr == NULL) {
786 KGSL_CORE_ERR("snapshot: no kernel mapping for global object 0x%016llX\n",
791 header->size = memdesc->size >> 2;
792 header->gpuaddr = memdesc->gpuaddr;
793 header->ptbase = MMU_DEFAULT_TTBR0(device);
794 header->type = SNAPSHOT_GPU_OBJECT_GLOBAL;
796 memcpy(ptr, memdesc->hostptr, memdesc->size);
798 return memdesc->size + sizeof(*header);
801 /* Snapshot IOMMU specific buffers */
802 static void adreno_snapshot_iommu(struct kgsl_device *device,
803 struct kgsl_snapshot *snapshot)
805 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
806 struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
808 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2,
809 snapshot, snapshot_global, &iommu->setstate);
811 if (ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION))
812 kgsl_snapshot_add_section(device,
813 KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2,
814 snapshot, snapshot_global, &iommu->smmu_info);
817 static void adreno_snapshot_ringbuffer(struct kgsl_device *device,
818 struct kgsl_snapshot *snapshot, struct adreno_ringbuffer *rb)
820 struct snapshot_rb_params params = {
821 .snapshot = snapshot,
828 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_RB_V2, snapshot,
829 snapshot_rb, ¶ms);
832 /* adreno_snapshot - Snapshot the Adreno GPU state
833 * @device - KGSL device to snapshot
834 * @snapshot - Pointer to the snapshot instance
835 * @context - context that caused the fault, if known by the driver
836 * This is a hook function called by kgsl_snapshot to snapshot the
837 * Adreno specific information for the GPU snapshot. In turn, this function
838 * calls the GPU specific snapshot function to get core specific information.
840 void adreno_snapshot(struct kgsl_device *device, struct kgsl_snapshot *snapshot,
841 struct kgsl_context *context)
844 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
845 struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
848 /* Reset the list of objects */
851 snapshot_frozen_objsize = 0;
853 setup_fault_process(device, snapshot,
854 context ? context->proc_priv : NULL);
856 adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB1_BASE,
857 ADRENO_REG_CP_IB1_BASE_HI, &snapshot->ib1base);
858 adreno_readreg(adreno_dev, ADRENO_REG_CP_IB1_BUFSZ, &snapshot->ib1size);
859 adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB2_BASE,
860 ADRENO_REG_CP_IB2_BASE_HI, &snapshot->ib2base);
861 adreno_readreg(adreno_dev, ADRENO_REG_CP_IB2_BUFSZ, &snapshot->ib2size);
863 snapshot->ib1dumped = false;
864 snapshot->ib2dumped = false;
866 adreno_snapshot_ringbuffer(device, snapshot, adreno_dev->cur_rb);
868 /* Dump the prev ringbuffer */
869 if (adreno_dev->prev_rb != adreno_dev->cur_rb)
870 adreno_snapshot_ringbuffer(device, snapshot,
871 adreno_dev->prev_rb);
873 if ((adreno_dev->next_rb != adreno_dev->prev_rb) &&
874 (adreno_dev->next_rb != adreno_dev->cur_rb))
875 adreno_snapshot_ringbuffer(device, snapshot,
876 adreno_dev->next_rb);
878 /* Add GPU specific sections - registers mainly, but other stuff too */
879 if (gpudev->snapshot)
880 gpudev->snapshot(adreno_dev, snapshot);
882 /* Dump selected global buffers */
883 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2,
884 snapshot, snapshot_global, &device->memstore);
886 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2,
887 snapshot, snapshot_global,
888 &adreno_dev->pwron_fixup);
890 if (kgsl_mmu_get_mmutype(device) == KGSL_MMU_TYPE_IOMMU)
891 adreno_snapshot_iommu(device, snapshot);
894 * Add a section that lists (gpuaddr, size, memtype) tuples of the
897 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_MEMLIST_V2,
898 snapshot, snapshot_capture_mem_list, snapshot->process);
900 * Make sure that the last IB1 that was being executed is dumped.
901 * Since this was the last IB1 that was processed, we should have
902 * already added it to the list during the ringbuffer parse but we
903 * want to be double plus sure.
904 * The problem is that IB size from the register is the unprocessed size
905 * of the buffer not the original size, so if we didn't catch this
906 * buffer being directly used in the RB, then we might not be able to
907 * dump the whole thing. Print a warning message so we can try to
908 * figure how often this really happens.
911 if (-ENOENT == find_object(snapshot->ib1base, snapshot->process) &&
913 kgsl_snapshot_push_object(snapshot->process, snapshot->ib1base,
916 "CP_IB1_BASE not found in the ringbuffer.Dumping %x dwords of the buffer.\n",
921 * Add the last parsed IB2 to the list. The IB2 should be found as we
922 * parse the objects below, but we try to add it to the list first, so
923 * it too can be parsed. Don't print an error message in this case - if
924 * the IB2 is found during parsing, the list will be updated with the
928 if (-ENOENT == find_object(snapshot->ib2base, snapshot->process)) {
929 kgsl_snapshot_push_object(snapshot->process, snapshot->ib2base,
934 * Go through the list of found objects and dump each one. As the IBs
935 * are parsed, more objects might be found, and objbufptr will increase
937 for (i = 0; i < objbufptr; i++)
938 dump_object(device, i, snapshot);
941 * Incase snapshot static blob is running out of memory, Add Active IB1
942 * and IB2 entries to obj_list so that active ib's can be dumped to
943 * snapshot dynamic blob.
945 if (!snapshot->ib1dumped || !snapshot->ib2dumped)
946 kgsl_snapshot_add_active_ib_obj_list(device, snapshot);
949 KGSL_CORE_ERR("Max objects found in IB\n");
950 if (snapshot_frozen_objsize)
951 KGSL_CORE_ERR("GPU snapshot froze %zdKb of GPU buffers\n",
952 snapshot_frozen_objsize / 1024);
957 * adreno_snapshot_cp_roq - Dump CP merciu data in snapshot
958 * @device: Device being snapshotted
959 * @remain: Bytes remaining in snapshot memory
960 * @priv: Size of merciu data in Dwords
962 size_t adreno_snapshot_cp_merciu(struct kgsl_device *device, u8 *buf,
963 size_t remain, void *priv)
965 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
966 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
967 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
968 int i, size = *((int *)priv);
970 /* The MERCIU data is two dwords per entry */
973 if (remain < DEBUG_SECTION_SZ(size)) {
974 SNAPSHOT_ERR_NOMEM(device, "CP MERCIU DEBUG");
978 header->type = SNAPSHOT_DEBUG_CP_MERCIU;
981 adreno_writereg(adreno_dev, ADRENO_REG_CP_MERCIU_ADDR, 0x0);
983 for (i = 0; i < size; i++) {
984 adreno_readreg(adreno_dev, ADRENO_REG_CP_MERCIU_DATA,
986 adreno_readreg(adreno_dev, ADRENO_REG_CP_MERCIU_DATA2,
990 return DEBUG_SECTION_SZ(size);
994 * adreno_snapshot_cp_roq - Dump ROQ data in snapshot
995 * @device: Device being snapshotted
996 * @remain: Bytes remaining in snapshot memory
997 * @priv: Size of ROQ data in Dwords
999 size_t adreno_snapshot_cp_roq(struct kgsl_device *device, u8 *buf,
1000 size_t remain, void *priv)
1002 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1003 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
1004 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
1005 int i, size = *((int *)priv);
1007 if (remain < DEBUG_SECTION_SZ(size)) {
1008 SNAPSHOT_ERR_NOMEM(device, "CP ROQ DEBUG");
1012 header->type = SNAPSHOT_DEBUG_CP_ROQ;
1013 header->size = size;
1015 adreno_writereg(adreno_dev, ADRENO_REG_CP_ROQ_ADDR, 0x0);
1016 for (i = 0; i < size; i++)
1017 adreno_readreg(adreno_dev, ADRENO_REG_CP_ROQ_DATA, &data[i]);
1019 return DEBUG_SECTION_SZ(size);
1023 * adreno_snapshot_cp_pm4_ram() - Dump PM4 data in snapshot
1024 * @device: Device being snapshotted
1025 * @buf: Snapshot memory
1026 * @remain: Number of bytes left in snapshot memory
1029 size_t adreno_snapshot_cp_pm4_ram(struct kgsl_device *device, u8 *buf,
1030 size_t remain, void *priv)
1032 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1033 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
1034 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
1036 size_t size = adreno_dev->pm4_fw_size - 1;
1038 if (remain < DEBUG_SECTION_SZ(size)) {
1039 SNAPSHOT_ERR_NOMEM(device, "CP PM4 RAM DEBUG");
1043 header->type = SNAPSHOT_DEBUG_CP_PM4_RAM;
1044 header->size = size;
1047 * Read the firmware from the GPU rather than use our cache in order to
1048 * try to catch mis-programming or corruption in the hardware. We do
1049 * use the cached version of the size, however, instead of trying to
1050 * maintain always changing hardcoded constants
1053 adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_RAM_RADDR, 0x0);
1054 for (i = 0; i < size; i++)
1055 adreno_readreg(adreno_dev, ADRENO_REG_CP_ME_RAM_DATA, &data[i]);
1057 return DEBUG_SECTION_SZ(size);
1061 * adreno_snapshot_cp_pfp_ram() - Dump the PFP data on snapshot
1062 * @device: Device being snapshotted
1063 * @buf: Snapshot memory
1064 * @remain: Amount of butes left in snapshot memory
1067 size_t adreno_snapshot_cp_pfp_ram(struct kgsl_device *device, u8 *buf,
1068 size_t remain, void *priv)
1070 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1071 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
1072 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
1073 int i, size = adreno_dev->pfp_fw_size - 1;
1075 if (remain < DEBUG_SECTION_SZ(size)) {
1076 SNAPSHOT_ERR_NOMEM(device, "CP PFP RAM DEBUG");
1080 header->type = SNAPSHOT_DEBUG_CP_PFP_RAM;
1081 header->size = size;
1084 * Read the firmware from the GPU rather than use our cache in order to
1085 * try to catch mis-programming or corruption in the hardware. We do
1086 * use the cached version of the size, however, instead of trying to
1087 * maintain always changing hardcoded constants
1089 adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR, 0x0);
1090 for (i = 0; i < size; i++)
1091 adreno_readreg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA,
1094 return DEBUG_SECTION_SZ(size);
1098 * adreno_snapshot_vpc_memory() - Save VPC data in snapshot
1099 * @device: Device being snapshotted
1100 * @buf: Snapshot memory
1101 * @remain: Number of bytes left in snapshot memory
1102 * @priv: Private data for VPC if any
1104 size_t adreno_snapshot_vpc_memory(struct kgsl_device *device, u8 *buf,
1105 size_t remain, void *priv)
1107 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1108 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
1109 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
1110 int vpc_mem_size = *((int *)priv);
1111 size_t size = VPC_MEMORY_BANKS * vpc_mem_size;
1112 int bank, addr, i = 0;
1114 if (remain < DEBUG_SECTION_SZ(size)) {
1115 SNAPSHOT_ERR_NOMEM(device, "VPC MEMORY");
1119 header->type = SNAPSHOT_DEBUG_VPC_MEMORY;
1120 header->size = size;
1122 for (bank = 0; bank < VPC_MEMORY_BANKS; bank++) {
1123 for (addr = 0; addr < vpc_mem_size; addr++) {
1124 unsigned int val = bank | (addr << 4);
1125 adreno_writereg(adreno_dev,
1126 ADRENO_REG_VPC_DEBUG_RAM_SEL, val);
1127 adreno_readreg(adreno_dev,
1128 ADRENO_REG_VPC_DEBUG_RAM_READ, &data[i++]);
1132 return DEBUG_SECTION_SZ(size);
1136 * adreno_snapshot_cp_meq() - Save CP MEQ data in snapshot
1137 * @device: Device being snapshotted
1138 * @buf: Snapshot memory
1139 * @remain: Number of bytes left in snapshot memory
1140 * @priv: Contains the size of MEQ data
1142 size_t adreno_snapshot_cp_meq(struct kgsl_device *device, u8 *buf,
1143 size_t remain, void *priv)
1145 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1146 struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
1147 unsigned int *data = (unsigned int *)(buf + sizeof(*header));
1149 int cp_meq_sz = *((int *)priv);
1151 if (remain < DEBUG_SECTION_SZ(cp_meq_sz)) {
1152 SNAPSHOT_ERR_NOMEM(device, "CP MEQ DEBUG");
1156 header->type = SNAPSHOT_DEBUG_CP_MEQ;
1157 header->size = cp_meq_sz;
1159 adreno_writereg(adreno_dev, ADRENO_REG_CP_MEQ_ADDR, 0x0);
1160 for (i = 0; i < cp_meq_sz; i++)
1161 adreno_readreg(adreno_dev, ADRENO_REG_CP_MEQ_DATA, &data[i]);
1163 return DEBUG_SECTION_SZ(cp_meq_sz);
1166 static const struct adreno_vbif_snapshot_registers *vbif_registers(
1167 struct adreno_device *adreno_dev,
1168 const struct adreno_vbif_snapshot_registers *list,
1171 unsigned int version;
1174 adreno_readreg(adreno_dev, ADRENO_REG_VBIF_VERSION, &version);
1176 for (i = 0; i < count; i++) {
1177 if ((list[i].version & list[i].mask) ==
1178 (version & list[i].mask))
1183 "snapshot: Registers for VBIF version %X register were not dumped\n",
1189 void adreno_snapshot_registers(struct kgsl_device *device,
1190 struct kgsl_snapshot *snapshot,
1191 const unsigned int *regs, unsigned int count)
1193 struct kgsl_snapshot_registers r;
1198 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot,
1199 kgsl_snapshot_dump_registers, &r);
1202 void adreno_snapshot_vbif_registers(struct kgsl_device *device,
1203 struct kgsl_snapshot *snapshot,
1204 const struct adreno_vbif_snapshot_registers *list,
1207 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
1208 struct kgsl_snapshot_registers regs;
1209 const struct adreno_vbif_snapshot_registers *vbif;
1211 vbif = vbif_registers(adreno_dev, list, count);
1214 regs.regs = vbif->registers;
1215 regs.count = vbif->count;
1217 kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS,
1218 snapshot, kgsl_snapshot_dump_registers, ®s);